diff --git a/crates/README.md b/crates/README.md index 3bfdbb0b5f..54922e3765 100644 --- a/crates/README.md +++ b/crates/README.md @@ -6,9 +6,9 @@ that can be tested and benchmarked with `cargo test` alone. | Crate | Responsibility | |---|---| -| `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, and a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte. | +| `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte, and run-cost allocation with exact Python `decimal` default-context arithmetic and Snowflake query-history SQL. | | `sqlbuild-cursor` | Microbatch cursor algebra: typed cursor values, grain flooring and stepping, half-open interval clamping, merging and batch splitting, timestamp text normalization and calendar durations, with Python `datetime` results and errors. Zones other than fixed offsets are resolved through a caller-supplied lookup. | -| `sqlbuild-state` | Append-only warehouse state rows: content-addressed migration event IDs, typed event SQL literals and statements, and source freshness data-version normalization, hashing, equivalence and age policy, byte-compatible with the rows Python wrote. | +| `sqlbuild-state` | Append-only warehouse state rows: content-addressed migration event IDs, typed event SQL literals and statements, and source freshness data-version normalization, hashing, equivalence, age policy and latest-state read SQL, byte-compatible with the rows Python wrote. | | `sqlbuild-pysource` | Static reading of project Python source with the ruff parser: macro module structure, literal SQL arguments, definition locations and string-literal edits. It never imports or runs user code; CPython stays the authority on whether a file is valid Python. | | `sqlbuild-cache` | The shared native store every native cache uses (one atomically replaced, checksummed file per cache kind, keyed by content digests) and the one fingerprinting scheme for cache keys: the content digest and the project file fingerprint. | | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution, static reference extraction, and the general reference scan with table-function call arguments under each adapter's lexical rules. | @@ -18,7 +18,7 @@ that can be tested and benchmarked with `cargo test` alone. | `sqlbuild-model-config` | MODEL configuration, columns and audits, template expansion over caller values, and dynamic-column name patterns using Rust regex syntax. | | `sqlbuild-render` | Macro call scanning, argument evaluation, expansion, splicing, memoisation and persistent call-store keys. Python executes user macro bodies at the extension boundary. | | `sqlbuild-attachments` | Native attachment of tests, audits, sources, functions, scenarios and seeds, including argument merging, validation and audit policy resolution. Adapter value rendering remains an extension boundary. | -| `sqlbuild-analysis` | SQL analysis over polyglot: query analysis, binding, semantic validation and usage, column references, type normalization, FAST/RICH lineage, and SQL-test/scenario extraction and compiler-side assembly. | +| `sqlbuild-analysis` | SQL analysis over polyglot: query analysis, binding, semantic validation and usage, column references, type normalization, FAST/RICH lineage, SQL-test/scenario extraction and compiler-side assembly, and Snowflake cost statement classification and query-tag parsing. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | | `sqlbuild-refactor` | Native rename/move plans, contract-adoption edits, MODEL/SCHEMA/YAML text edits and atomic staging/commit with rollback. Shared scope placement delegates to the native scope implementation; user Python source inspection remains at the Python AST boundary. | | `sqlbuild-cli` | The `sqb` and `sqlbuild` executables: `--version`, unchanged compiles replayed from the shared reuse store with Python's exact streams, exit code and compute log capture, and a handoff of every other invocation to the installed `sqlbuild-python` script. Built as its own wheel (`crates/sqlbuild-cli/pyproject.toml`). | diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/setting_help.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/setting_help.rs index b3e3de3d5e..ef77d39392 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/setting_help.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/setting_help.rs @@ -1,4 +1,4 @@ -//! Python's setting help texts (`sqlbuild.errors.setting_help`) for string settings. +//! Setting help texts for string settings, matching the shared native setting help. use crate::contracts::constants::{ADDITIONAL_HELP_SEPARATOR, SETTING_SNIPPET_INDENT}; diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 79b7b528c6..1b29ef8804 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -38,6 +38,7 @@ pub mod resource_diagnostics; pub mod semantic_checks; pub mod semantic_usage; pub mod semantic_validation; +pub mod snowflake_cost; pub mod sql_test_attachment; pub mod sql_tokens; pub mod target_paths; diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/mod.rs b/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/mod.rs new file mode 100644 index 0000000000..c0d723475b --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod statements; diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs b/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs new file mode 100644 index 0000000000..7e49d9c53d --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs @@ -0,0 +1,43 @@ +//! Query-type prefixes, Python whitespace and CREATE statement parsing. + +use polyglot_sql::{Dialect, DialectType, Expression, ParseOptions}; + +use crate::snowflake_cost::constants::{CREATE_VARIANT_PREFIX, DYNAMIC_TABLE_MODIFIER}; + +pub(crate) fn of_kind(query_type: &str, kind: &str) -> bool { + query_type == kind + || query_type + .strip_prefix(kind) + .is_some_and(|rest| rest.starts_with('_')) +} + +/// Python's `str.isspace` for one character. +pub(crate) fn python_space(character: char) -> bool { + character.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&character) +} + +/// Whether a CREATE statement's text parses as a definition that needs no warehouse. +pub(crate) fn metadata_only_create(query_text: &str) -> bool { + let options: ParseOptions = ParseOptions { + complexity_guard: None, + }; + let Ok(mut statements) = + Dialect::get(DialectType::Snowflake).parse_with_options(query_text, &options) + else { + return false; + }; + if statements.len() != 1 { + return false; + } + match statements.remove(0) { + Expression::CreateTable(table) => { + !(table.as_select.is_some() + || table.clone_source.is_some() + || table.using_template.is_some() + || table.table_modifier.as_deref().unwrap_or("").to_uppercase() + == DYNAMIC_TABLE_MODIFIER) + } + Expression::CreateView(view) if view.materialized => false, + statement => statement.variant_name().starts_with(CREATE_VARIANT_PREFIX), + } +} diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs b/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs new file mode 100644 index 0000000000..e941f13be5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs @@ -0,0 +1,10 @@ +//! Snowflake query types, table modifiers and the SQLBuild query-tag identity. + +pub(crate) const USE_QUERY_TYPE: &str = "USE"; +pub(crate) const DROP_QUERY_TYPE: &str = "DROP"; +pub(crate) const CREATE_QUERY_TYPE: &str = "CREATE"; +pub(crate) const DYNAMIC_TABLE_MODIFIER: &str = "DYNAMIC"; +pub(crate) const CREATE_VARIANT_PREFIX: &str = "create"; +pub(crate) const QUERY_TAG_APP: &str = "sqlbuild"; +pub(crate) const QUERY_TAG_APP_KEY: &str = "app"; +pub(crate) const QUERY_TAG_VERSION_KEY: &str = "v"; diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/main/metadata_only_statement.rs b/crates/sqlbuild-analysis/src/snowflake_cost/main/metadata_only_statement.rs new file mode 100644 index 0000000000..05c21b691e --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/main/metadata_only_statement.rs @@ -0,0 +1,19 @@ +//! `metadata_only_statement`: Snowflake statements that run without warehouse compute. + +use crate::snowflake_cost::_helpers::statements::{metadata_only_create, of_kind, python_space}; +use crate::snowflake_cost::constants::{CREATE_QUERY_TYPE, DROP_QUERY_TYPE, USE_QUERY_TYPE}; + +/// Identify statements Snowflake executes without warehouse compute. +pub fn metadata_only_statement(query_type: &str, query_text: Option<&str>) -> bool { + let normalized: String = query_type.to_uppercase().replace(' ', "_"); + if of_kind(&normalized, USE_QUERY_TYPE) || of_kind(&normalized, DROP_QUERY_TYPE) { + return true; + } + if !of_kind(&normalized, CREATE_QUERY_TYPE) { + return false; + } + match query_text { + Some(text) if !text.chars().all(python_space) => metadata_only_create(text), + _ => false, + } +} diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/main/mod.rs b/crates/sqlbuild-analysis/src/snowflake_cost/main/mod.rs new file mode 100644 index 0000000000..cf473b47aa --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/main/mod.rs @@ -0,0 +1,2 @@ +pub mod metadata_only_statement; +pub mod sqlbuild_query_tag; diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/main/sqlbuild_query_tag.rs b/crates/sqlbuild-analysis/src/snowflake_cost/main/sqlbuild_query_tag.rs new file mode 100644 index 0000000000..497ffa35c7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/main/sqlbuild_query_tag.rs @@ -0,0 +1,46 @@ +//! `sqlbuild_query_tag`: the run attribution of a SQLBuild Snowflake `QUERY_TAG`. + +use serde_json::Value; + +use crate::snowflake_cost::constants::{QUERY_TAG_APP, QUERY_TAG_APP_KEY, QUERY_TAG_VERSION_KEY}; +use crate::snowflake_cost::models::QueryTag; + +/// Python's `value == 1` for a decoded JSON value. +fn equals_one(value: Option<&Value>) -> bool { + match value { + Some(Value::Bool(flag)) => *flag, + Some(Value::Number(number)) => number + .as_i64() + .map(|integer| integer == 1) + .or_else(|| number.as_u64().map(|integer| integer == 1)) + .unwrap_or_else(|| number.as_f64() == Some(1.0)), + _ => false, + } +} + +fn text(payload: &serde_json::Map, key: &str) -> Option { + match payload.get(key) { + Some(Value::String(value)) if !value.is_empty() => Some(value.clone()), + _ => None, + } +} + +/// The tag's attribution when it is a SQLBuild version-1 JSON object, else `None`. +pub fn sqlbuild_query_tag(value: &str) -> Option { + if value.is_empty() { + return None; + } + let Ok(Value::Object(payload)) = serde_json::from_str::(value) else { + return None; + }; + let app_matches: bool = + matches!(payload.get(QUERY_TAG_APP_KEY), Some(Value::String(app)) if app == QUERY_TAG_APP); + if !app_matches || !equals_one(payload.get(QUERY_TAG_VERSION_KEY)) { + return None; + } + Some(QueryTag { + run_id: text(&payload, "run_id"), + resource_type: text(&payload, "resource_type"), + resource_name: text(&payload, "resource_name"), + }) +} diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/mod.rs b/crates/sqlbuild-analysis/src/snowflake_cost/mod.rs new file mode 100644 index 0000000000..1e64f3d116 --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/models.rs b/crates/sqlbuild-analysis/src/snowflake_cost/models.rs new file mode 100644 index 0000000000..a3188938cc --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/models.rs @@ -0,0 +1,9 @@ +//! The run attribution a SQLBuild Snowflake query tag carries. + +/// Non-empty string fields of a SQLBuild query tag. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct QueryTag { + pub run_id: Option, + pub resource_type: Option, + pub resource_name: Option, +} diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/tests.rs b/crates/sqlbuild-analysis/src/snowflake_cost/tests.rs new file mode 100644 index 0000000000..79d552fda0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_snowflake_cost.rs"] +mod snowflake_cost; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/tests/test_snowflake_cost.rs b/crates/sqlbuild-analysis/src/snowflake_cost/tests/test_snowflake_cost.rs new file mode 100644 index 0000000000..c373af2531 --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/tests/test_snowflake_cost.rs @@ -0,0 +1,94 @@ +use crate::snowflake_cost::main::metadata_only_statement::metadata_only_statement; +use crate::snowflake_cost::main::sqlbuild_query_tag::sqlbuild_query_tag; +use crate::snowflake_cost::models::QueryTag; +use crate::snowflake_cost::tests::test_types::{MetadataOnlyTestCase, QueryTagTestCase}; + +#[test] +fn given_snowflake_statements_when_classifying_then_detects_metadata_only_work() { + let test_cases = [ + MetadataOnlyTestCase { + description: "use statements need no warehouse", + query_type: "USE_DATABASE", + query_text: None, + expected_metadata_only: true, + }, + MetadataOnlyTestCase { + description: "a column-list table is metadata only", + query_type: "CREATE_TABLE", + query_text: Some("CREATE TABLE t (a INT)"), + expected_metadata_only: true, + }, + MetadataOnlyTestCase { + description: "a table built from a query uses the warehouse", + query_type: "CREATE_TABLE", + query_text: Some("CREATE TABLE t AS SELECT 1 AS a"), + expected_metadata_only: false, + }, + MetadataOnlyTestCase { + description: "a materialized view uses the warehouse", + query_type: "CREATE_VIEW", + query_text: Some("CREATE MATERIALIZED VIEW v AS SELECT 1 AS a"), + expected_metadata_only: false, + }, + MetadataOnlyTestCase { + description: "blank create text is not classified", + query_type: "CREATE", + query_text: Some(" "), + expected_metadata_only: false, + }, + MetadataOnlyTestCase { + description: "a create-prefixed type without separator is ordinary work", + query_type: "CREATEX", + query_text: Some("CREATE TABLE t (a INT)"), + expected_metadata_only: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + metadata_only_statement(test_case.query_type, test_case.query_text), + test_case.expected_metadata_only, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_query_tags_when_parsing_then_reads_sqlbuild_attribution() { + let test_cases = [ + QueryTagTestCase { + description: "a version-one tag keeps non-empty strings", + value: r#"{"app": "sqlbuild", "v": 1.0, "run_id": "r", "resource_name": ""}"#, + expected_tag: Some(QueryTag { + run_id: Some("r".to_owned()), + resource_type: None, + resource_name: None, + }), + }, + QueryTagTestCase { + description: "true equals one", + value: r#"{"app": "sqlbuild", "v": true}"#, + expected_tag: Some(QueryTag::default()), + }, + QueryTagTestCase { + description: "other versions are ignored", + value: r#"{"app": "sqlbuild", "v": 2}"#, + expected_tag: None, + }, + QueryTagTestCase { + description: "non-objects are ignored", + value: "[]", + expected_tag: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + sqlbuild_query_tag(test_case.value), + test_case.expected_tag, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/tests/test_types.rs b/crates/sqlbuild-analysis/src/snowflake_cost/tests/test_types.rs new file mode 100644 index 0000000000..20b8acfeca --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/tests/test_types.rs @@ -0,0 +1,14 @@ +use crate::snowflake_cost::models::QueryTag; + +pub(crate) struct MetadataOnlyTestCase { + pub(crate) description: &'static str, + pub(crate) query_type: &'static str, + pub(crate) query_text: Option<&'static str>, + pub(crate) expected_metadata_only: bool, +} + +pub(crate) struct QueryTagTestCase { + pub(crate) description: &'static str, + pub(crate) value: &'static str, + pub(crate) expected_tag: Option, +} diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs index 3fd864dcbd..969eeb4ed6 100644 --- a/crates/sqlbuild-config/src/command_connection/constants.rs +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -22,3 +22,35 @@ pub(crate) const UNSUPPORTED_WAREHOUSE_CODE: &str = "C260"; pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; /// Two quotes around at least one character. pub(crate) const MIN_QUOTED_IDENTIFIER_LENGTH: usize = 3; +/// The warehouse group each CLI command selects a target's default warehouse from. +pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 29] = [ + ("audit", Some("build")), + ("build", Some("build")), + ("check", Some("build")), + ("clone", Some("build")), + ("dbt", Some("build")), + ("load", Some("build")), + ("plan", Some("build")), + ("seed", Some("build")), + ("contract", Some("query")), + ("cost", Some("query")), + ("debug", Some("query")), + ("diff", Some("query")), + ("freshness", Some("query")), + ("query", Some("query")), + ("janitor", Some("query")), + ("scenario", Some("query")), + ("test", Some("query")), + ("clean", None), + ("compile", None), + ("dag", None), + ("format", None), + ("init", None), + ("lineage", None), + ("mv", None), + ("playground", None), + ("rename", None), + ("rules", None), + ("scope", None), + ("skills", None), +]; diff --git a/crates/sqlbuild-config/src/project/_helpers/help.rs b/crates/sqlbuild-config/src/project/_helpers/help.rs index 14bf4a9fe6..9cdb500837 100644 --- a/crates/sqlbuild-config/src/project/_helpers/help.rs +++ b/crates/sqlbuild-config/src/project/_helpers/help.rs @@ -13,6 +13,8 @@ use crate::project::errors::ConfigFailure; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum SettingValue<'a> { Bool(bool), + /// An integer, as its decimal text. + Integer(&'a str), Text(&'a str), } @@ -29,6 +31,7 @@ fn toml_value(value: SettingValue<'_>) -> String { match value { SettingValue::Bool(true) => "true".to_owned(), SettingValue::Bool(false) => "false".to_owned(), + SettingValue::Integer(digits) => digits.to_owned(), SettingValue::Text(text) => { format!("\"{}\"", text.replace('\\', "\\\\").replace('"', "\\\"")) } diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs index 6a2fc8fe75..a3e65d82e8 100644 --- a/crates/sqlbuild-config/src/project/main/mod.rs +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -4,5 +4,7 @@ pub mod load_project_config; pub mod local_sql_analysis_failure; pub mod microbatch_guidance; pub mod resolve_config_file; +pub mod setting_help_text; +pub mod snippet_help_text; pub mod sql_analysis_off_guidance; pub mod unsupported_target_warehouse; diff --git a/crates/sqlbuild-config/src/project/main/setting_help_text.rs b/crates/sqlbuild-config/src/project/main/setting_help_text.rs new file mode 100644 index 0000000000..734be6fbc4 --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/setting_help_text.rs @@ -0,0 +1,23 @@ +//! Help that shows the exact setting to write. + +use crate::project::_helpers::help::{Setting, SettingValue, setting_help}; +use crate::project::models::{SettingSite, SettingText}; + +/// `, set this in :` followed by the exact TOML section and key. +#[must_use] +pub fn setting_help_text(purpose: &str, site: SettingSite<'_>, value: &SettingText) -> String { + let value: SettingValue<'_> = match value { + SettingText::Bool(flag) => SettingValue::Bool(*flag), + SettingText::Integer(digits) => SettingValue::Integer(digits), + SettingText::Text(text) => SettingValue::Text(text), + }; + setting_help( + purpose, + &Setting { + file_name: site.file_name, + section: site.section, + key: site.key, + value, + }, + ) +} diff --git a/crates/sqlbuild-config/src/project/main/snippet_help_text.rs b/crates/sqlbuild-config/src/project/main/snippet_help_text.rs new file mode 100644 index 0000000000..c21bb6bc3c --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/snippet_help_text.rs @@ -0,0 +1,13 @@ +//! Help that shows an exact authored snippet to add. + +use crate::project::constants::SETTING_SNIPPET_INDENT; + +/// `, :` followed by the exact lines to write, one per line. +#[must_use] +pub fn snippet_help_text(purpose: &str, target: &str, lines: &[String]) -> String { + let snippet: Vec = lines + .iter() + .map(|line| format!("{SETTING_SNIPPET_INDENT}{line}")) + .collect(); + format!("{purpose}, {target}:\n{}", snippet.join("\n")) +} diff --git a/crates/sqlbuild-config/src/project/models.rs b/crates/sqlbuild-config/src/project/models.rs index 2874ba17bb..ff2a018aa6 100644 --- a/crates/sqlbuild-config/src/project/models.rs +++ b/crates/sqlbuild-config/src/project/models.rs @@ -38,3 +38,20 @@ pub enum Built { /// `Decimal(str(value))` of an integer or float. Decimal(ConfigValue), } + +/// A setting value as help text shows it in TOML. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SettingText { + Bool(bool), + /// An integer, as its decimal text. + Integer(String), + Text(String), +} + +/// Where a setting lives: its file name, TOML section and key. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SettingSite<'a> { + pub file_name: &'a str, + pub section: &'a str, + pub key: &'a str, +} diff --git a/crates/sqlbuild-config/src/project/tests/test_guidance.rs b/crates/sqlbuild-config/src/project/tests/test_guidance.rs index fe68a81e20..f8277f5fb6 100644 --- a/crates/sqlbuild-config/src/project/tests/test_guidance.rs +++ b/crates/sqlbuild-config/src/project/tests/test_guidance.rs @@ -1,6 +1,9 @@ use crate::project::main::explicit_references_help::explicit_references_help; use crate::project::main::microbatch_guidance::microbatch_guidance; +use crate::project::main::setting_help_text::setting_help_text; +use crate::project::main::snippet_help_text::snippet_help_text; use crate::project::main::sql_analysis_off_guidance::sql_analysis_off_guidance; +use crate::project::models::{SettingSite, SettingText}; use crate::project::tests::test_types::GuidanceTestCase; #[test] @@ -24,6 +27,47 @@ fn given_setting_guidance_when_rendering_then_text_matches_python() { expected_note: "the current value is [references] enforce_explicit = true (from sqlbuild_project.toml or its default); while migrating a project, to allow X, set this in sqlbuild_project.toml:\n [references]\n enforce_explicit = false", expected_help: "", }, + GuidanceTestCase { + description: "a quoted string setting and an integer setting", + rendered: ( + setting_help_text( + "to stage", + SettingSite { + file_name: "sqlbuild_project.toml", + section: "settings", + key: "table_promotion_mode", + }, + &SettingText::Text("st\"aged\\".to_owned()), + ), + setting_help_text( + "to wait", + SettingSite { + file_name: "sqlbuild_local.toml", + section: "settings", + key: "timeout", + }, + &SettingText::Integer("30".to_owned()), + ), + ), + expected_note: "to stage, set this in sqlbuild_project.toml:\n [settings]\n table_promotion_mode = \"st\\\"aged\\\\\"", + expected_help: "to wait, set this in sqlbuild_local.toml:\n [settings]\n timeout = 30", + }, + GuidanceTestCase { + description: "a snippet help shows every line to write", + rendered: ( + snippet_help_text( + "to describe it", + "add this to seeds/lookups.yml", + &[ + "- name: product_types".to_owned(), + " description: Types".to_owned(), + ], + ), + String::new(), + ), + expected_note: "to describe it, add this to seeds/lookups.yml:\n - name: product_types\n description: Types", + expected_help: "", + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-core/src/build_command/constants.rs b/crates/sqlbuild-core/src/build_command/constants.rs index e4672a7a50..a7f88bc604 100644 --- a/crates/sqlbuild-core/src/build_command/constants.rs +++ b/crates/sqlbuild-core/src/build_command/constants.rs @@ -53,3 +53,19 @@ pub(crate) const PYTHON_NODE_FAILED: &str = "failed"; pub(crate) const PYTHON_STATUS_OK: &str = "OK"; pub(crate) const PYTHON_STATUS_SKIP: &str = "SKIP"; pub(crate) const PYTHON_STATUS_FAIL: &str = "FAIL"; + +pub const MISSING_ORIGIN_BUILD_HELP: &str = + "Pass --allow-missing-migration-origin to confirm in non-interactive runs."; +pub const MISSING_ORIGIN_DBT_HELP: &str = "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected models with sqb build --allow-missing-migration-origin and leave them out of this dbt selection, or set the target's missing_migration_origin to 'allow'."; +pub const SNAPSHOT_FULL_REFRESH_BUILD_HELP: &str = + "Pass --allow-snapshot-full-refresh to confirm in non-interactive runs."; +pub const SNAPSHOT_FULL_REFRESH_DBT_HELP: &str = "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected snapshots with sqb build --allow-snapshot-full-refresh and leave them out of this dbt selection, or relax the snapshot_full_refresh policy."; +pub const TABLE_TYPE_DOWNGRADE_BUILD_HELP: &str = + "Pass --allow-table-type-downgrade to confirm in non-interactive runs."; +pub const TABLE_TYPE_DOWNGRADE_DBT_HELP: &str = "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected models with sqb build --allow-table-type-downgrade and leave them out of this dbt selection, or set the target's table_type_downgrade to 'allow'."; +pub const RETENTION_DECREASE_BUILD_HELP: &str = + "Pass --allow-retention-decrease to confirm in non-interactive runs."; +pub const RETENTION_DECREASE_DBT_HELP: &str = "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected models with sqb build --allow-retention-decrease and leave them out of this dbt selection, or set the target's time_travel_retention_decrease to 'allow'."; +pub const EXECUTION_LIMIT_BUILD_NOTE: &str = "No warehouse changes were made."; +pub const EXECUTION_LIMIT_DBT_NOTE: &str = + "dbt work has already run; no SQLBuild models were built."; diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index bb689cf0d2..a7edb05016 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -85,6 +85,21 @@ impl CliStyle { self.apply(BOLD, text) } + #[must_use] + pub fn dbt_object_name(self, text: &str) -> String { + self.apply("", text) + } + + #[must_use] + pub fn dbt_execution_label(self, text: &str) -> String { + self.apply(ORANGE, text) + } + + #[must_use] + pub fn log_label(self, text: &str) -> String { + self.apply(DIM, text) + } + #[must_use] pub fn success(self, text: &str) -> String { self.apply(GREEN, text) @@ -616,3 +631,34 @@ pub struct MigrationTextReport { pub display: DisplayLimit, pub manual_rename_hint: String, } + +/// One planned table-type conversion line. +#[derive(Clone, Debug)] +pub struct TableTypeText { + pub model_name: String, + pub desired_type: String, + pub actual_type: Option, + pub source: String, + pub irreversible_warning: Option, +} + +/// One planned retention change line. +#[derive(Clone, Debug)] +pub struct RetentionText { + pub scope: String, + pub desired_days: i64, + pub actual_days: Option, + pub effective_days: Option, + pub source: String, + pub direction: String, + pub phase: String, + pub irreversible_warning: Option, +} + +/// Storage work planned before model builds, ahead of the migration sections. +#[derive(Clone, Debug)] +pub struct PreBuildWorkText { + pub table_types: Vec, + pub retention: Vec, + pub display: DisplayLimit, +} diff --git a/crates/sqlbuild-core/src/cost_allocation/_helpers/allocation.rs b/crates/sqlbuild-core/src/cost_allocation/_helpers/allocation.rs new file mode 100644 index 0000000000..7dccb5e13f --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/_helpers/allocation.rs @@ -0,0 +1,119 @@ +//! Busy-time attribution, warehouse rates, totals and ordering of priced resources. + +use std::cmp::Ordering; +use std::collections::{BTreeSet, HashMap}; + +use crate::cost_allocation::_helpers::decimal::{ + add, compare, divide, from_float, integer, multiply, +}; +use crate::cost_allocation::constants::{CREDITS_PER_HOUR, MICROS_PER_SECOND, SECONDS_PER_HOUR}; +use crate::cost_allocation::models::{CostQuery, PyDecimal, ResourceAllocation}; +use crate::cost_allocation::types::ResourceKey; + +/// Items grouped by key in first-appearance order, as a Python `defaultdict(list)`. +pub(crate) fn grouped<'a, K: Eq + std::hash::Hash + Clone>( + items: impl Iterator, + key: impl Fn(&CostQuery) -> K, +) -> Vec<(K, Vec<&'a CostQuery>)> { + let mut positions: HashMap = HashMap::new(); + let mut groups: Vec<(K, Vec<&'a CostQuery>)> = Vec::new(); + for item in items { + let item_key: K = key(item); + match positions.get(&item_key) { + Some(&position) => groups[position].1.push(item), + None => { + positions.insert(item_key.clone(), groups.len()); + groups.push((item_key, vec![item])); + } + } + } + groups +} + +/// Busy seconds attributed to each run query ID, accumulated in Python's iteration order. +pub(crate) fn attributed_seconds(queries: &[CostQuery]) -> HashMap { + let mut attributed: HashMap = HashMap::new(); + for (_, warehouse_queries) in grouped(queries.iter(), |query| query.warehouse.clone()) { + let boundaries: Vec = warehouse_queries + .iter() + .flat_map(|query| [query.started_micros, query.completed_micros]) + .collect::>() + .into_iter() + .collect(); + for window in boundaries.windows(2) { + let (start, end) = (window[0], window[1]); + let active: Vec<&&CostQuery> = warehouse_queries + .iter() + .filter(|query| query.started_micros < end && query.completed_micros > start) + .collect(); + if active.is_empty() { + continue; + } + let elapsed: f64 = (end - start) as f64 / MICROS_PER_SECOND; + let Some(elapsed) = from_float(elapsed) else { + continue; + }; + let share: PyDecimal = divide(&elapsed, active.len() as u64); + for query in active.into_iter().filter(|query| query.in_run) { + let total: PyDecimal = match attributed.get(&query.query_id) { + Some(total) => add(total, &share), + None => add(&integer(0), &share), + }; + attributed.insert(query.query_id.clone(), total); + } + } + } + attributed +} + +pub(crate) fn credits_per_hour(size: &str) -> Option { + let size: String = size.to_uppercase(); + CREDITS_PER_HOUR + .iter() + .find(|(name, _)| *name == size) + .map(|(_, rate)| *rate) +} + +pub(crate) fn sum<'a>(values: impl Iterator) -> PyDecimal { + values.fold(integer(0), |total, value| add(&total, value)) +} + +/// Python's sort key of a resource cost: most expensive first, then by identity. +pub(crate) fn resource_order(left: &ResourceAllocation, right: &ResourceAllocation) -> Ordering { + let cluster = |resource: &ResourceAllocation| resource.warehouse.cluster.unwrap_or(-1); + compare(&right.estimated_usd, &left.estimated_usd) + .then_with(|| left.resource_name.cmp(&right.resource_name)) + .then_with(|| left.resource_type.cmp(&right.resource_type)) + .then_with(|| left.warehouse.name.cmp(&right.warehouse.name)) + .then_with(|| left.warehouse.size.cmp(&right.warehouse.size)) + .then_with(|| { + left.warehouse + .kind + .as_deref() + .unwrap_or("") + .cmp(right.warehouse.kind.as_deref().unwrap_or("")) + }) + .then_with(|| cluster(left).cmp(&cluster(right))) +} + +pub(crate) fn resource_allocation( + (resource_type, resource_name, warehouse): ResourceKey, + queries: &[&CostQuery], + (attributed, rate, usd_per_credit): (&HashMap, u64, &PyDecimal), +) -> ResourceAllocation { + let zero: PyDecimal = integer(0); + let seconds: PyDecimal = sum(queries + .iter() + .map(|query| attributed.get(&query.query_id).unwrap_or(&zero))); + let credits: PyDecimal = divide(&multiply(&seconds, &integer(rate)), SECONDS_PER_HOUR); + ResourceAllocation { + resource_type, + resource_name, + warehouse, + query_count: queries.len(), + attributed_seconds: seconds, + bytes_scanned: queries.iter().map(|query| query.bytes_scanned).sum(), + estimated_usd: multiply(&credits, usd_per_credit), + estimated_compute_credits: credits, + } +} diff --git a/crates/sqlbuild-core/src/cost_allocation/_helpers/decimal.rs b/crates/sqlbuild-core/src/cost_allocation/_helpers/decimal.rs new file mode 100644 index 0000000000..5ce0ff4df4 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/_helpers/decimal.rs @@ -0,0 +1,279 @@ +//! Python `Decimal` arithmetic in the default context: 28 digits, `ROUND_HALF_EVEN`. + +use std::cmp::Ordering; + +#[cfg(test)] +use crate::cost_allocation::constants::SMALLEST_PLAIN_LEFT_DIGITS; +use crate::cost_allocation::constants::{DECIMAL_PRECISION, HALF_DIGIT, MAX_DIGIT}; +use crate::cost_allocation::models::PyDecimal; +use crate::json::main::python_float_repr::python_float_repr; + +/// Digits without leading zeros; zero is `[0]`. +fn normalized(mut digits: Vec) -> Vec { + let leading: usize = digits.iter().take_while(|digit| **digit == 0).count(); + digits.drain(..leading.min(digits.len().saturating_sub(1))); + if digits.is_empty() { + digits.push(0); + } + digits +} + +fn is_zero(digits: &[u8]) -> bool { + digits.iter().all(|digit| *digit == 0) +} + +/// An exact integer as a decimal with exponent zero. +pub(crate) fn integer(value: u64) -> PyDecimal { + PyDecimal { + digits: value.to_string().bytes().map(|byte| byte - b'0').collect(), + exponent: 0, + } +} + +/// A signed decimal exponent such as `+16` or `-06`. +fn exponent_value(text: &str) -> Option { + let (negative, digits): (bool, &str) = match text.as_bytes().first() { + Some(b'-') => (true, &text[1..]), + Some(b'+') => (false, &text[1..]), + _ => (false, text), + }; + if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + let magnitude: i64 = digits.bytes().fold(0_i64, |value, byte| { + value + .saturating_mul(10) + .saturating_add(i64::from(byte - b'0')) + }); + Some(if negative { -magnitude } else { magnitude }) +} + +/// `Decimal(text)` for plain decimal or scientific text of a non-negative finite number. +pub(crate) fn parse(text: &str) -> Option { + let (mantissa, exponent): (&str, i64) = match text.find(['e', 'E']) { + Some(index) => (&text[..index], exponent_value(&text[index + 1..])?), + None => (text, 0), + }; + let (whole, fraction): (&str, &str) = mantissa.split_once('.').unwrap_or((mantissa, "")); + if whole.is_empty() && fraction.is_empty() { + return None; + } + let digits: Vec = whole + .bytes() + .chain(fraction.bytes()) + .map(|byte| byte.is_ascii_digit().then(|| byte - b'0')) + .collect::>()?; + let fraction_length: i64 = i64::try_from(fraction.len()).unwrap_or(i64::MAX); + Some(PyDecimal { + digits: normalized(digits), + exponent: exponent - fraction_length, + }) +} + +/// Round to the context precision with `ROUND_HALF_EVEN`, as `Decimal._fix` does. +pub(crate) fn rounded(digits: Vec, exponent: i64) -> PyDecimal { + let digits: Vec = normalized(digits); + if digits.len() <= DECIMAL_PRECISION { + return PyDecimal { digits, exponent }; + } + let dropped: usize = digits.len() - DECIMAL_PRECISION; + let mut kept: Vec = digits[..DECIMAL_PRECISION].to_vec(); + let rest: &[u8] = &digits[DECIMAL_PRECISION..]; + let round_up: bool = match rest[0].cmp(&HALF_DIGIT) { + Ordering::Greater => true, + Ordering::Less => false, + Ordering::Equal => !is_zero(&rest[1..]) || kept[DECIMAL_PRECISION - 1] % 2 == 1, + }; + let mut exponent: i64 = exponent + i64::try_from(dropped).unwrap_or(i64::MAX); + if round_up { + let mut index: usize = kept.len(); + loop { + if index == 0 { + kept.insert(0, 1); + break; + } + index -= 1; + if kept[index] == MAX_DIGIT { + kept[index] = 0; + } else { + kept[index] += 1; + break; + } + } + if kept.len() > DECIMAL_PRECISION { + kept.pop(); + exponent += 1; + } + } + PyDecimal { + digits: kept, + exponent, + } +} + +fn shifted(value: &PyDecimal, exponent: i64) -> Vec { + let zeros: usize = usize::try_from(value.exponent - exponent).unwrap_or(0); + let mut digits: Vec = value.digits.clone(); + digits.extend(std::iter::repeat_n(0, zeros)); + digits +} + +/// `left + right`. +pub(crate) fn add(left: &PyDecimal, right: &PyDecimal) -> PyDecimal { + let exponent: i64 = left.exponent.min(right.exponent); + let (mut first, mut second) = (shifted(left, exponent), shifted(right, exponent)); + let width: usize = first.len().max(second.len()) + 1; + first.splice(0..0, std::iter::repeat_n(0, width - first.len())); + second.splice(0..0, std::iter::repeat_n(0, width - second.len())); + let mut carry: u8 = 0; + let mut sum: Vec = vec![0; width]; + for index in (0..width).rev() { + let total: u8 = first[index] + second[index] + carry; + sum[index] = total % 10; + carry = total / 10; + } + rounded(sum, exponent) +} + +/// `left * right`. +pub(crate) fn multiply(left: &PyDecimal, right: &PyDecimal) -> PyDecimal { + let mut product: Vec = vec![0; left.digits.len() + right.digits.len()]; + for (left_index, left_digit) in left.digits.iter().enumerate().rev() { + for (right_index, right_digit) in right.digits.iter().enumerate().rev() { + product[left_index + right_index + 1] += + u32::from(*left_digit) * u32::from(*right_digit); + } + } + for index in (1..product.len()).rev() { + product[index - 1] += product[index] / 10; + product[index] %= 10; + } + let digits: Vec = product + .into_iter() + .map(|digit| u8::try_from(digit).unwrap_or(0)) + .collect(); + rounded(digits, left.exponent + right.exponent) +} + +/// Quotient and remainder of a digit string divided by a small positive integer. +fn divide_digits(digits: &[u8], divisor: u64) -> (Vec, bool) { + let mut remainder: u128 = 0; + let mut quotient: Vec = Vec::with_capacity(digits.len()); + for digit in digits { + remainder = remainder * 10 + u128::from(*digit); + quotient.push(u8::try_from(remainder / u128::from(divisor)).unwrap_or(0)); + remainder %= u128::from(divisor); + } + (normalized(quotient), remainder != 0) +} + +fn last_digit(digits: &[u8]) -> u8 { + digits.last().copied().unwrap_or(0) +} + +/// `dividend / Decimal(divisor)` for a positive integer divisor, as `Decimal.__truediv__`. +pub(crate) fn divide(dividend: &PyDecimal, divisor: u64) -> PyDecimal { + if is_zero(÷nd.digits) { + return PyDecimal { + digits: vec![0], + exponent: dividend.exponent, + }; + } + let divisor_length: i64 = i64::try_from(divisor.to_string().len()).unwrap_or(0); + let dividend_length: i64 = i64::try_from(dividend.digits.len()).unwrap_or(0); + let precision: i64 = i64::try_from(DECIMAL_PRECISION).unwrap_or(0); + let shift: i64 = divisor_length - dividend_length + precision + 1; + let mut exponent: i64 = dividend.exponent - shift; + let (mut quotient, inexact) = if shift >= 0 { + let mut digits: Vec = dividend.digits.clone(); + digits.extend(std::iter::repeat_n(0, usize::try_from(shift).unwrap_or(0))); + divide_digits(&digits, divisor) + } else { + let split: usize = dividend.digits.len() - usize::try_from(-shift).unwrap_or(0); + let (high, low) = dividend.digits.split_at(split); + let (quotient, remainder) = divide_digits(high, divisor); + (quotient, remainder || !is_zero(low)) + }; + if inexact { + if last_digit("ient).is_multiple_of(HALF_DIGIT) { + let index: usize = quotient.len() - 1; + quotient[index] += 1; + } + } else { + while exponent < dividend.exponent && quotient.len() > 1 && last_digit("ient) == 0 { + quotient.pop(); + exponent += 1; + } + } + rounded(quotient, exponent) +} + +/// Numeric order of two decimals, ignoring their exponents' representation. +pub(crate) fn compare(left: &PyDecimal, right: &PyDecimal) -> Ordering { + match (is_zero(&left.digits), is_zero(&right.digits)) { + (true, true) => return Ordering::Equal, + (true, false) => return Ordering::Less, + (false, true) => return Ordering::Greater, + (false, false) => {} + } + let adjusted = |value: &PyDecimal| -> i64 { + value.exponent + i64::try_from(value.digits.len()).unwrap_or(0) + }; + adjusted(left).cmp(&adjusted(right)).then_with(|| { + let width: usize = left.digits.len().max(right.digits.len()); + let padded = |digits: &[u8]| -> Vec { + let mut padded: Vec = digits.to_vec(); + padded.resize(width, 0); + padded + }; + padded(&left.digits).cmp(&padded(&right.digits)) + }) +} + +/// `str(Decimal)`: scientific notation when the exponent is positive or the value is tiny. +#[cfg(test)] +pub(crate) fn text(value: &PyDecimal) -> String { + let digits: String = value + .digits + .iter() + .map(|digit| char::from(b'0' + digit)) + .collect(); + let length: i64 = i64::try_from(digits.len()).unwrap_or(0); + let left_digits: i64 = value.exponent + length; + let dot_place: i64 = if value.exponent <= 0 && left_digits > SMALLEST_PLAIN_LEFT_DIGITS { + left_digits + } else { + 1 + }; + let (integer_part, fraction_part): (String, String) = if dot_place <= 0 { + ( + "0".to_owned(), + format!( + ".{}{digits}", + "0".repeat(usize::try_from(-dot_place).unwrap_or(0)) + ), + ) + } else if dot_place >= length { + ( + format!( + "{digits}{}", + "0".repeat(usize::try_from(dot_place - length).unwrap_or(0)) + ), + String::new(), + ) + } else { + let split: usize = usize::try_from(dot_place).unwrap_or(0); + (digits[..split].to_owned(), format!(".{}", &digits[split..])) + }; + let suffix: String = if left_digits == dot_place { + String::new() + } else { + format!("E{:+}", left_digits - dot_place) + }; + format!("{integer_part}{fraction_part}{suffix}") +} + +/// `Decimal(str(value))` for a non-negative finite float; `None` for NaN or infinities. +pub(crate) fn from_float(value: f64) -> Option { + parse(&python_float_repr(value)) +} diff --git a/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs b/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs new file mode 100644 index 0000000000..3b9c8a22a1 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod allocation; +pub(crate) mod decimal; diff --git a/crates/sqlbuild-core/src/cost_allocation/constants.rs b/crates/sqlbuild-core/src/cost_allocation/constants.rs new file mode 100644 index 0000000000..09c13bd940 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/constants.rs @@ -0,0 +1,34 @@ +//! Python `decimal` context limits, Snowflake credit rates and allocation messages. + +/// `decimal.getcontext().prec` of the default context. +pub(crate) const DECIMAL_PRECISION: usize = 28; +pub(crate) const SECONDS_PER_HOUR: u64 = 3_600; +pub(crate) const MICROS_PER_SECOND: f64 = 1_000_000.0; +pub(crate) const STANDARD_WAREHOUSE_TYPE: &str = "STANDARD"; +pub(crate) const UNKNOWN_RESOURCE: &str = "unknown"; +pub(crate) const RESULT_LIMIT_LIMITATION: &str = + "Snowflake query-history result limit was reached."; +/// Credits per hour by uppercased Snowflake warehouse size. +pub(crate) const CREDITS_PER_HOUR: [(&str, u64); 10] = [ + ("X-SMALL", 1), + ("SMALL", 2), + ("MEDIUM", 4), + ("LARGE", 8), + ("X-LARGE", 16), + ("2X-LARGE", 32), + ("3X-LARGE", 64), + ("4X-LARGE", 128), + ("5X-LARGE", 256), + ("6X-LARGE", 512), +]; +/// The default `RESULT_LIMIT` of one Snowflake query-history read. +pub const QUERY_HISTORY_RESULT_LIMIT: usize = 10_000; +pub(crate) const QUERY_HISTORY_FUNCTION: &str = "INFORMATION_SCHEMA.QUERY_HISTORY"; +pub(crate) const NULL_QUERY_TEXT: &str = "NULL AS QUERY_TEXT"; +/// The rounding digit at which `ROUND_HALF_EVEN` looks past the first dropped digit. +pub(crate) const HALF_DIGIT: u8 = 5; +/// `str(Decimal)` switches to exponent form at or below this many left digits. +#[cfg(test)] +pub(crate) const SMALLEST_PLAIN_LEFT_DIGITS: i64 = -6; +/// The largest decimal digit, which carries when rounded up. +pub(crate) const MAX_DIGIT: u8 = 9; diff --git a/crates/sqlbuild-core/src/cost_allocation/main/allocate_run_cost.rs b/crates/sqlbuild-core/src/cost_allocation/main/allocate_run_cost.rs new file mode 100644 index 0000000000..ba5ea60444 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/main/allocate_run_cost.rs @@ -0,0 +1,104 @@ +//! `allocate_run_cost`: share each warehouse's busy time equally among concurrently active queries. + +use std::collections::{BTreeSet, HashMap}; + +use crate::cost_allocation::_helpers::allocation::{ + attributed_seconds, credits_per_hour, grouped, resource_allocation, resource_order, sum, +}; +use crate::cost_allocation::_helpers::decimal::integer; +use crate::cost_allocation::constants::{ + RESULT_LIMIT_LIMITATION, STANDARD_WAREHOUSE_TYPE, UNKNOWN_RESOURCE, +}; +use crate::cost_allocation::models::{ + AllocationStatus, CostQuery, PyDecimal, ResourceAllocation, RunAllocation, WarehouseKey, +}; + +/// Attribute the run's share of busy warehouse time and price it; pending without run queries. +pub fn allocate_run_cost( + queries: &[CostQuery], + usd_per_credit: &PyDecimal, + result_limit_reached: bool, +) -> RunAllocation { + let run_queries: Vec<&CostQuery> = queries.iter().filter(|query| query.in_run).collect(); + if run_queries.is_empty() { + return RunAllocation { + status: AllocationStatus::Pending, + resources: Vec::new(), + query_count: 0, + attributed_seconds: integer(0), + bytes_scanned: 0, + estimated_compute_credits: integer(0), + estimated_usd: integer(0), + limitations: Vec::new(), + }; + } + let attributed: HashMap = attributed_seconds(queries); + let mut limitations: Vec = Vec::new(); + let mut resources: Vec = Vec::new(); + for (key, group) in grouped(run_queries.iter().copied(), |query| { + ( + query + .resource_type + .clone() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| UNKNOWN_RESOURCE.to_owned()), + query + .resource_name + .clone() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| UNKNOWN_RESOURCE.to_owned()), + query.warehouse.clone(), + ) + }) { + let warehouse: &WarehouseKey = &key.2; + let rate: Option = credits_per_hour(&warehouse.size); + match &warehouse.kind { + None => limitations.push(format!("Unknown warehouse type for {}", warehouse.name)), + Some(kind) if kind.to_uppercase() != STANDARD_WAREHOUSE_TYPE => { + limitations.push(format!("Unsupported Snowflake warehouse type: {kind}")); + continue; + } + Some(_) => {} + } + if warehouse.cluster.is_none() { + limitations.push(format!("Unknown cluster number for {}", warehouse.name)); + } + let Some(rate) = rate else { + limitations.push(format!( + "Unknown Snowflake warehouse size: {}", + warehouse.size + )); + continue; + }; + resources.push(resource_allocation( + key, + &group, + (&attributed, rate, usd_per_credit), + )); + } + if result_limit_reached { + limitations.push(RESULT_LIMIT_LIMITATION.to_owned()); + } + resources.sort_by(resource_order); + let limitations: Vec = limitations + .into_iter() + .collect::>() + .into_iter() + .collect(); + RunAllocation { + status: if limitations.is_empty() { + AllocationStatus::Complete + } else { + AllocationStatus::Partial + }, + query_count: run_queries.len(), + attributed_seconds: sum(resources.iter().map(|item| &item.attributed_seconds)), + bytes_scanned: resources.iter().map(|item| item.bytes_scanned).sum(), + estimated_compute_credits: sum(resources + .iter() + .map(|item| &item.estimated_compute_credits)), + estimated_usd: sum(resources.iter().map(|item| &item.estimated_usd)), + limitations, + resources, + } +} diff --git a/crates/sqlbuild-core/src/cost_allocation/main/mod.rs b/crates/sqlbuild-core/src/cost_allocation/main/mod.rs new file mode 100644 index 0000000000..d4b3364262 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/main/mod.rs @@ -0,0 +1,2 @@ +pub mod allocate_run_cost; +pub mod query_history_sql; diff --git a/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs b/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs new file mode 100644 index 0000000000..40b6028249 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs @@ -0,0 +1,66 @@ +//! `query_history_sql`: the Snowflake `QUERY_HISTORY` read of one end-time window. + +use crate::cost_allocation::constants::{NULL_QUERY_TEXT, QUERY_HISTORY_FUNCTION}; +use crate::cost_allocation::models::QueryHistoryWindow; + +fn query_history_function(database: Option<&str>) -> String { + match database { + None => QUERY_HISTORY_FUNCTION.to_owned(), + Some(database) => format!( + "\"{}\".{QUERY_HISTORY_FUNCTION}", + database.to_uppercase().replace('"', "\"\"") + ), + } +} + +fn query_text_expression(query_ids: &[String]) -> String { + if query_ids.is_empty() { + return NULL_QUERY_TEXT.to_owned(); + } + let mut sorted: Vec<&String> = query_ids.iter().collect(); + sorted.sort(); + sorted.dedup(); + let literals: Vec = sorted + .into_iter() + .map(|query_id| format!("'{}'", query_id.replace('\'', "''"))) + .collect(); + format!( + "CASE WHEN QUERY_ID IN ({}) AND (QUERY_TYPE = 'CREATE' OR QUERY_TYPE LIKE 'CREATE_%') \ +THEN QUERY_TEXT END AS QUERY_TEXT", + literals.join(", ") + ) +} + +/// Query-history SQL for queries ending after `start`, and before `end` when it is bounded. +pub fn query_history_sql(window: &QueryHistoryWindow<'_>) -> String { + let end_range: String = match window.end { + Some(end) => format!("\n END_TIME_RANGE_END => TO_TIMESTAMP_LTZ('{end}'),"), + None => String::new(), + }; + format!( + "SELECT + QUERY_ID, + QUERY_TAG, + WAREHOUSE_NAME, + WAREHOUSE_SIZE, + WAREHOUSE_TYPE, + CLUSTER_NUMBER, + START_TIME, + END_TIME, + EXECUTION_TIME, + BYTES_SCANNED, + EXECUTION_STATUS, + QUERY_TYPE, + {}, + CURRENT_TIMESTAMP() +FROM TABLE({}( + END_TIME_RANGE_START => TO_TIMESTAMP_LTZ('{}'),{end_range} + RESULT_LIMIT => {} +)) +ORDER BY START_TIME, QUERY_ID", + query_text_expression(window.classification_query_ids), + query_history_function(window.database), + window.start, + window.result_limit, + ) +} diff --git a/crates/sqlbuild-core/src/cost_allocation/mod.rs b/crates/sqlbuild-core/src/cost_allocation/mod.rs new file mode 100644 index 0000000000..5853224abe --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub(crate) mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/cost_allocation/models.rs b/crates/sqlbuild-core/src/cost_allocation/models.rs new file mode 100644 index 0000000000..349f862fbe --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/models.rs @@ -0,0 +1,74 @@ +//! Exact Python decimals, observed queries and their attributed run cost. + +/// A finite non-negative Python `Decimal`: coefficient digits, most significant first, and exponent. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PyDecimal { + pub digits: Vec, + pub exponent: i64, +} + +/// The warehouse identity whose busy time concurrent queries share. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct WarehouseKey { + pub name: String, + pub size: String, + pub kind: Option, + pub cluster: Option, +} + +/// One observed query; instants are microseconds on one consistent clock. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CostQuery { + pub query_id: String, + pub warehouse: WarehouseKey, + pub started_micros: i128, + pub completed_micros: i128, + pub in_run: bool, + pub resource_type: Option, + pub resource_name: Option, + pub bytes_scanned: i128, +} + +/// The cost attributed to one resource on one warehouse (`ResourceCost`). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResourceAllocation { + pub resource_type: String, + pub resource_name: String, + pub warehouse: WarehouseKey, + pub query_count: usize, + pub attributed_seconds: PyDecimal, + pub bytes_scanned: i128, + pub estimated_compute_credits: PyDecimal, + pub estimated_usd: PyDecimal, +} + +/// Whether a run's queries were seen and fully priced. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AllocationStatus { + Pending, + Partial, + Complete, +} + +/// The run cost of the selected run's queries (`RunCostSummary` without its rate fields). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RunAllocation { + pub status: AllocationStatus, + pub resources: Vec, + pub query_count: usize, + pub attributed_seconds: PyDecimal, + pub bytes_scanned: i128, + pub estimated_compute_credits: PyDecimal, + pub estimated_usd: PyDecimal, + pub limitations: Vec, +} + +/// The time window and classified query IDs of one query-history read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct QueryHistoryWindow<'a> { + pub database: Option<&'a str>, + pub start: &'a str, + pub end: Option<&'a str>, + pub classification_query_ids: &'a [String], + pub result_limit: usize, +} diff --git a/crates/sqlbuild-core/src/cost_allocation/tests.rs b/crates/sqlbuild-core/src/cost_allocation/tests.rs new file mode 100644 index 0000000000..6949838ed8 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_allocation.rs"] +mod allocation; +#[path = "tests/test_decimal.rs"] +mod decimal; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/cost_allocation/tests/helpers.rs b/crates/sqlbuild-core/src/cost_allocation/tests/helpers.rs new file mode 100644 index 0000000000..ec1a534bf5 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/tests/helpers.rs @@ -0,0 +1,28 @@ +use crate::cost_allocation::_helpers::decimal::{from_float, integer, parse}; +use crate::cost_allocation::models::{CostQuery, PyDecimal, WarehouseKey}; + +pub(crate) fn decimal(text: &str) -> PyDecimal { + parse(text).unwrap_or_else(|| integer(0)) +} + +pub(crate) fn float(value: f64) -> PyDecimal { + from_float(value).unwrap_or_else(|| integer(0)) +} + +pub(crate) fn query(index: u64) -> CostQuery { + CostQuery { + query_id: format!("q{index}"), + warehouse: WarehouseKey { + name: "WH".to_owned(), + size: "X-SMALL".to_owned(), + kind: Some("STANDARD".to_owned()), + cluster: Some(1), + }, + started_micros: 0, + completed_micros: 1_000_000, + in_run: true, + resource_type: Some("model".to_owned()), + resource_name: Some("orders".to_owned()), + bytes_scanned: 1, + } +} diff --git a/crates/sqlbuild-core/src/cost_allocation/tests/test_allocation.rs b/crates/sqlbuild-core/src/cost_allocation/tests/test_allocation.rs new file mode 100644 index 0000000000..6d9d540098 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/tests/test_allocation.rs @@ -0,0 +1,45 @@ +use crate::cost_allocation::_helpers::decimal::text; +use crate::cost_allocation::main::allocate_run_cost::allocate_run_cost; +use crate::cost_allocation::models::{CostQuery, PyDecimal, RunAllocation}; +use crate::cost_allocation::tests::helpers::query; +use crate::cost_allocation::tests::test_types::AllocationTestCase; + +#[test] +fn given_concurrent_queries_when_allocating_then_shares_busy_seconds_equally() { + let test_cases = [ + AllocationTestCase { + description: "one query owns the whole second", + concurrent_queries: 1, + expected_seconds: "1.0", + expected_usd: "0.0008333333333333333333333333334", + }, + AllocationTestCase { + description: "seven concurrent queries round each share", + concurrent_queries: 7, + expected_seconds: "1.000000000000000000000000000", + expected_usd: "0.0008333333333333333333333333334", + }, + ]; + + for test_case in test_cases { + let queries: Vec = (0..test_case.concurrent_queries).map(query).collect(); + let rate: PyDecimal = PyDecimal { + digits: vec![3, 0, 0], + exponent: -2, + }; + let allocation: RunAllocation = allocate_run_cost(&queries, &rate, false); + + assert_eq!( + ( + text(&allocation.attributed_seconds), + text(&allocation.estimated_usd), + ), + ( + test_case.expected_seconds.to_owned(), + test_case.expected_usd.to_owned(), + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/cost_allocation/tests/test_decimal.rs b/crates/sqlbuild-core/src/cost_allocation/tests/test_decimal.rs new file mode 100644 index 0000000000..356b9ce1ca --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/tests/test_decimal.rs @@ -0,0 +1,86 @@ +use crate::cost_allocation::_helpers::decimal::{add, divide, integer, multiply, text}; +use crate::cost_allocation::tests::helpers::{decimal, float}; +use crate::cost_allocation::tests::test_types::DecimalOperationTestCase; + +#[test] +fn given_python_decimals_when_operating_then_rounds_like_the_default_context() { + let test_cases = [ + DecimalOperationTestCase { + description: "inexact division rounds to 28 digits", + result: divide(&decimal("1.0"), 3), + expected_text: "0.3333333333333333333333333333", + }, + DecimalOperationTestCase { + description: "tiny quotients switch to scientific text", + result: divide(&decimal("0.000005"), 7), + expected_text: "7.142857142857142857142857143E-7", + }, + DecimalOperationTestCase { + description: "exact division keeps the ideal exponent", + result: divide(&decimal("12"), 4), + expected_text: "3", + }, + DecimalOperationTestCase { + description: "exact division keeps needed fraction digits", + result: divide(&decimal("7.5"), 3), + expected_text: "2.5", + }, + DecimalOperationTestCase { + description: "zero division keeps the dividend exponent", + result: divide(&decimal("0.000000"), 3_600), + expected_text: "0.000000", + }, + DecimalOperationTestCase { + description: "adding zero aligns a large exponent and rounds", + result: add(&decimal("1E+30"), &integer(0)), + expected_text: "1.000000000000000000000000000E+30", + }, + DecimalOperationTestCase { + description: "a half above an odd coefficient rounds up with carry", + result: add(&decimal("9999999999999999999999999999"), &decimal("0.5")), + expected_text: "1.000000000000000000000000000E+28", + }, + DecimalOperationTestCase { + description: "a half above an even coefficient rounds to even", + result: add(&decimal("9999999999999999999999999998"), &decimal("0.5")), + expected_text: "9999999999999999999999999998", + }, + DecimalOperationTestCase { + description: "long products round to 28 digits", + result: multiply( + &decimal("3.123456789012345678901234567890123"), + &decimal("1.234"), + ), + expected_text: "3.854345677641234567764123457", + }, + DecimalOperationTestCase { + description: "zero products keep the summed exponent", + result: multiply(&integer(0), &decimal("3.00")), + expected_text: "0.00", + }, + DecimalOperationTestCase { + description: "integral floats keep repr's trailing zero", + result: float(3_600.0), + expected_text: "3600.0", + }, + DecimalOperationTestCase { + description: "small floats read repr's exponent form", + result: float(5e-6), + expected_text: "0.000005", + }, + DecimalOperationTestCase { + description: "large floats read repr's exponent form", + result: float(1e16), + expected_text: "1E+16", + }, + ]; + + for test_case in test_cases { + assert_eq!( + text(&test_case.result), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/cost_allocation/tests/test_types.rs b/crates/sqlbuild-core/src/cost_allocation/tests/test_types.rs new file mode 100644 index 0000000000..cefbc507fe --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/tests/test_types.rs @@ -0,0 +1,14 @@ +use crate::cost_allocation::models::PyDecimal; + +pub(crate) struct DecimalOperationTestCase { + pub(crate) description: &'static str, + pub(crate) result: PyDecimal, + pub(crate) expected_text: &'static str, +} + +pub(crate) struct AllocationTestCase { + pub(crate) description: &'static str, + pub(crate) concurrent_queries: u64, + pub(crate) expected_seconds: &'static str, + pub(crate) expected_usd: &'static str, +} diff --git a/crates/sqlbuild-core/src/cost_allocation/types.rs b/crates/sqlbuild-core/src/cost_allocation/types.rs new file mode 100644 index 0000000000..2936fd82bc --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/types.rs @@ -0,0 +1,6 @@ +//! Grouping keys of run-cost allocation. + +use crate::cost_allocation::models::WarehouseKey; + +/// Resource type, resource name and warehouse of one priced resource. +pub(crate) type ResourceKey = (String, String, WarehouseKey); diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index a363d58d5c..2e9cf4e654 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -10,6 +10,7 @@ pub mod cli_text; pub mod clone_text; pub mod constants; pub mod contract_command; +pub mod cost_allocation; pub mod cost_command; pub mod cost_report; pub mod dbt_cli; diff --git a/crates/sqlbuild-core/src/plan_display/constants.rs b/crates/sqlbuild-core/src/plan_display/constants.rs index 0445540d36..aa6211db6f 100644 --- a/crates/sqlbuild-core/src/plan_display/constants.rs +++ b/crates/sqlbuild-core/src/plan_display/constants.rs @@ -33,3 +33,5 @@ pub(crate) const MIGRATION_MANUAL: &str = "manual"; pub(crate) const MIGRATION_AUTOMATIC: &str = "automatic"; pub(crate) const MIGRATION_INCOMPATIBLE: &str = "incompatible"; pub(crate) const WATERMARK_STRATEGY: &str = "watermark"; +pub(crate) const QUERY_DIFF_INDENT: &str = " "; +pub(crate) const UNIFIED_DIFF_HEADER_PREFIXES: [&str; 3] = ["---", "+++", "@@"]; diff --git a/crates/sqlbuild-core/src/plan_display/main/format_pre_build_work.rs b/crates/sqlbuild-core/src/plan_display/main/format_pre_build_work.rs new file mode 100644 index 0000000000..05f76d251a --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/format_pre_build_work.rs @@ -0,0 +1,57 @@ +//! Table-type and retention work that runs before model builds. + +use crate::cli_text::main::format_overflow_line::format_overflow_line; +use crate::cli_text::models::PreBuildWorkText; + +fn days(value: Option) -> String { + value.map_or_else(|| "missing".to_owned(), |days| format!("{days}d")) +} + +/// Append table-type conversions, with an overflow hint, and every retention change. +#[must_use] +pub fn format_pre_build_work(mut lines: Vec, work: &PreBuildWorkText) -> Vec { + if !work.table_types.is_empty() { + lines.push("Table type conversions".to_owned()); + let visible = work + .display + .max_entries + .map_or(work.table_types.len(), |limit| { + limit.min(work.table_types.len()) + }); + for entry in &work.table_types[..visible] { + lines.push(format!( + " {} desired={} actual={} source={}", + entry.model_name, + entry.desired_type, + entry.actual_type.as_deref().unwrap_or("unknown"), + entry.source + )); + if let Some(warning) = &entry.irreversible_warning { + lines.push(format!(" WARNING: {warning}")); + } + } + let hidden = i64::try_from(work.table_types.len() - visible).unwrap_or(i64::MAX); + if let Some(line) = format_overflow_line(hidden, " ", &work.display.overflow_flag) { + lines.push(line); + } + } + if !work.retention.is_empty() { + lines.push("Retention".to_owned()); + } + for entry in &work.retention { + lines.push(format!( + " {} desired={}d actual={} effective={} source={} direction={} phase={}", + entry.scope, + entry.desired_days, + days(entry.actual_days), + days(entry.effective_days), + entry.source, + entry.direction, + entry.phase + )); + if let Some(warning) = &entry.irreversible_warning { + lines.push(format!(" WARNING: {warning}")); + } + } + lines +} diff --git a/crates/sqlbuild-core/src/plan_display/main/format_query_diff.rs b/crates/sqlbuild-core/src/plan_display/main/format_query_diff.rs new file mode 100644 index 0000000000..321c688b44 --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/format_query_diff.rs @@ -0,0 +1,26 @@ +//! Coloured, indented unified-diff lines for a changed model query. + +use crate::cli_text::models::CliStyle; +use crate::plan_display::constants::{QUERY_DIFF_INDENT, UNIFIED_DIFF_HEADER_PREFIXES}; + +/// Indent each unified-diff line and colour additions, removals and headers. +#[must_use] +pub fn format_query_diff(diff_lines: &[String], style: CliStyle) -> Vec { + let mut formatted_lines: Vec = Vec::with_capacity(diff_lines.len()); + for line in diff_lines { + let formatted = format!("{QUERY_DIFF_INDENT}{line}"); + let header = UNIFIED_DIFF_HEADER_PREFIXES + .iter() + .any(|prefix| line.starts_with(prefix)); + formatted_lines.push(if line.starts_with('+') && !line.starts_with("+++") { + style.success(&formatted) + } else if line.starts_with('-') && !line.starts_with("---") { + style.error(&formatted) + } else if header { + style.muted(&formatted) + } else { + formatted + }); + } + formatted_lines +} diff --git a/crates/sqlbuild-core/src/plan_display/main/mod.rs b/crates/sqlbuild-core/src/plan_display/main/mod.rs index c9d5735ad4..ab3ac3c3a5 100644 --- a/crates/sqlbuild-core/src/plan_display/main/mod.rs +++ b/crates/sqlbuild-core/src/plan_display/main/mod.rs @@ -3,6 +3,8 @@ pub mod effective_microbatch_batch_size; pub mod effective_timestamp_grain; pub mod format_model_migrations; pub mod format_no_input_rows; +pub mod format_pre_build_work; +pub mod format_query_diff; pub mod format_waiting_on_empty_inputs; pub mod model_execution_annotation; pub mod model_materialization_label; diff --git a/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs b/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs index 5e41e9cb63..012be8076b 100644 --- a/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs +++ b/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs @@ -1,7 +1,12 @@ +use crate::cli_text::models::{ + CliStyle, DisplayLimit, PreBuildWorkText, RetentionText, TableTypeText, +}; use crate::plan_display::main::cursor_bound_display::cursor_bound_display; use crate::plan_display::main::effective_microbatch_batch_size::effective_microbatch_batch_size; use crate::plan_display::main::effective_timestamp_grain::effective_timestamp_grain; use crate::plan_display::main::format_no_input_rows::format_no_input_rows; +use crate::plan_display::main::format_pre_build_work::format_pre_build_work; +use crate::plan_display::main::format_query_diff::format_query_diff; use crate::plan_display::main::format_waiting_on_empty_inputs::format_waiting_on_empty_inputs; use crate::plan_display::main::model_execution_annotation::model_execution_annotation; use crate::plan_display::main::model_materialization_label::model_materialization_label; @@ -9,7 +14,8 @@ use crate::plan_display::main::model_resource_type::model_resource_type; use crate::plan_display::models::{ModelLabelFacts, ParsedBound, TimestampGrainFacts}; use crate::plan_display::tests::helpers::{facts, timestamp_grains}; use crate::plan_display::tests::test_types::{ - BatchSizeTestCase, BoundTestCase, LabelTestCase, NamesTestCase, TimestampGrainTestCase, + BatchSizeTestCase, BoundTestCase, LabelTestCase, NamesTestCase, PreBuildWorkTestCase, + QueryDiffTestCase, TimestampGrainTestCase, }; #[test] @@ -260,3 +266,116 @@ fn given_cursor_grains_when_resolving_replay_grain_then_coarsest_timestamp_grain ); } } + +#[test] +fn given_storage_work_when_formatting_pre_build_work_then_matches_python_text() { + let table_type = TableTypeText { + model_name: "orders".into(), + desired_type: "transient".into(), + actual_type: None, + source: "model".into(), + irreversible_warning: Some("drops history".into()), + }; + let retention = RetentionText { + scope: "analytics.orders".into(), + desired_days: 7, + actual_days: Some(1), + effective_days: None, + source: "project".into(), + direction: "increase".into(), + phase: "pre_build".into(), + irreversible_warning: None, + }; + let test_cases = [ + PreBuildWorkTestCase { + description: "lists every entry without a limit", + work: PreBuildWorkText { + table_types: vec![table_type.clone()], + retention: vec![retention], + display: DisplayLimit { + max_entries: None, + overflow_flag: "--verbose".into(), + }, + }, + expected_lines: vec![ + "Table type conversions", + " orders desired=transient actual=unknown source=model", + " WARNING: drops history", + "Retention", + " analytics.orders desired=7d actual=1d effective=missing source=project direction=increase phase=pre_build", + ], + }, + PreBuildWorkTestCase { + description: "limits table types with an overflow hint", + work: PreBuildWorkText { + table_types: vec![table_type.clone(), table_type], + retention: vec![], + display: DisplayLimit { + max_entries: Some(0), + overflow_flag: "--verbose".into(), + }, + }, + expected_lines: vec![ + "Table type conversions", + "\u{1b}[2m ... and 2 more (use --verbose to show all)\u{1b}[0m", + ], + }, + ]; + for test_case in test_cases { + assert_eq!( + format_pre_build_work(Vec::new(), &test_case.work), + test_case.expected_lines, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_unified_diff_when_formatting_query_diff_then_lines_are_indented_and_coloured() { + let diff: Vec = [ + "--- previous", + "+++ current", + "@@ -1 +1 @@", + "-a", + "+b", + " c", + ] + .into_iter() + .map(String::from) + .collect(); + let test_cases = [ + QueryDiffTestCase { + description: "plain diff is only indented", + style: CliStyle { use_color: false }, + expected_lines: vec![ + " --- previous", + " +++ current", + " @@ -1 +1 @@", + " -a", + " +b", + " c", + ], + }, + QueryDiffTestCase { + description: "coloured diff marks additions, removals and headers", + style: CliStyle { use_color: true }, + expected_lines: vec![ + "\x1b[2m --- previous\x1b[0m", + "\x1b[2m +++ current\x1b[0m", + "\x1b[2m @@ -1 +1 @@\x1b[0m", + "\x1b[38;5;167m -a\x1b[0m", + "\x1b[32m +b\x1b[0m", + " c", + ], + }, + ]; + for test_case in test_cases { + assert_eq!( + format_query_diff(&diff, test_case.style), + test_case.expected_lines, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/plan_display/tests/test_types.rs b/crates/sqlbuild-core/src/plan_display/tests/test_types.rs index d924f3052d..30466be5cf 100644 --- a/crates/sqlbuild-core/src/plan_display/tests/test_types.rs +++ b/crates/sqlbuild-core/src/plan_display/tests/test_types.rs @@ -1,3 +1,4 @@ +use crate::cli_text::models::{CliStyle, PreBuildWorkText}; use crate::plan_display::models::{ModelLabelFacts, ParsedBound, TimestampGrainFacts}; pub(super) struct LabelTestCase { @@ -35,3 +36,15 @@ pub(super) struct TimestampGrainTestCase { pub(super) facts: TimestampGrainFacts, pub(super) expected_grain: Result, &'static str>, } + +pub(super) struct PreBuildWorkTestCase { + pub(super) description: &'static str, + pub(super) work: PreBuildWorkText, + pub(super) expected_lines: Vec<&'static str>, +} + +pub(super) struct QueryDiffTestCase { + pub(super) description: &'static str, + pub(super) style: CliStyle, + pub(super) expected_lines: Vec<&'static str>, +} diff --git a/crates/sqlbuild-cursor/src/planner/main/cursor_end_clamp.rs b/crates/sqlbuild-cursor/src/planner/main/cursor_end_clamp.rs new file mode 100644 index 0000000000..11f8ff8396 --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/main/cursor_end_clamp.rs @@ -0,0 +1,25 @@ +//! Clamping a concrete cursor window to an exclusive configured `cursor_end`. + +use std::cmp::Ordering; + +use crate::algebra::errors::CursorError; +use crate::algebra::main::compare::compare; +use crate::algebra::models::Scalar; +use crate::algebra::types::ZoneRules; +use crate::planner::models::CursorEndClamp; + +/// How a `[start, end)` window changes so it never reaches past `cursor_end`. +pub fn cursor_end_clamp( + window: (&Scalar, &Scalar), + cursor_end: &Scalar, + rules: &impl ZoneRules, +) -> Result { + let (start, end) = window; + if compare(start, cursor_end, rules)? != Ordering::Less { + return Ok(CursorEndClamp::Empty); + } + if compare(end, cursor_end, rules)? == Ordering::Greater { + return Ok(CursorEndClamp::Capped); + } + Ok(CursorEndClamp::Unchanged) +} diff --git a/crates/sqlbuild-cursor/src/planner/main/mod.rs b/crates/sqlbuild-cursor/src/planner/main/mod.rs index 9c989bded1..ca5f77b454 100644 --- a/crates/sqlbuild-cursor/src/planner/main/mod.rs +++ b/crates/sqlbuild-cursor/src/planner/main/mod.rs @@ -1,8 +1,10 @@ //! Planner cursor algebra entry points. +pub mod cursor_end_clamp; pub mod eligible_start; pub mod empty_inputs; pub mod override_kind; pub mod raw_bounds; pub mod replay_start; +pub mod rolling_window; pub mod safety_horizon; diff --git a/crates/sqlbuild-cursor/src/planner/main/rolling_window.rs b/crates/sqlbuild-cursor/src/planner/main/rolling_window.rs new file mode 100644 index 0000000000..c4501e570c --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/main/rolling_window.rs @@ -0,0 +1,44 @@ +//! Invocation-anchored rolling microbatch window. + +use crate::algebra::errors::CursorError; +use crate::algebra::main::floor_to_grain::floor_to_grain; +use crate::algebra::main::shift_by_duration::shift_by_duration; +use crate::algebra::models::{DurationSpan, Grain, Moment, Scalar, ScalarValue}; + +/// The `[clock floored to grain - span, clock floored to grain)` window of a UTC clock. +pub fn rolling_window( + clock: &Scalar, + span: &DurationSpan, + grain: Grain, +) -> Result<(Scalar, Scalar), CursorError> { + let anchor: Scalar = floor_to_grain(clock, Some(grain)); + let ScalarValue::Timestamp(anchored) = &anchor.value else { + return Err(CursorError::Type( + "invocation clock must be a timestamp".into(), + )); + }; + let Moment::Timestamp { local, fold } = shift_by_duration( + Moment::Timestamp { + local: anchored.local, + fold: anchored.fold, + }, + span, + -1, + )? + else { + return Err(CursorError::Type( + "invocation clock must be a timestamp".into(), + )); + }; + let mut start = anchored.clone(); + start.local = local; + start.fold = fold; + start.source_text = None; + Ok(( + Scalar { + value: ScalarValue::Timestamp(start), + handle: None, + }, + anchor, + )) +} diff --git a/crates/sqlbuild-cursor/src/planner/models.rs b/crates/sqlbuild-cursor/src/planner/models.rs index 0c51018bcc..570af6b417 100644 --- a/crates/sqlbuild-cursor/src/planner/models.rs +++ b/crates/sqlbuild-cursor/src/planner/models.rs @@ -31,3 +31,14 @@ pub struct ReplayPolicy { pub has_backfill: bool, pub has_start_override: bool, } + +/// The change a configured exclusive `cursor_end` makes to a concrete window. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CursorEndClamp { + /// The window already ends at or before `cursor_end`. + Unchanged, + /// The window starts at or after `cursor_end`, so it collapses to `[cursor_end, cursor_end)`. + Empty, + /// The window ends after `cursor_end`, so its end becomes `cursor_end`. + Capped, +} diff --git a/crates/sqlbuild-cursor/src/planner/tests.rs b/crates/sqlbuild-cursor/src/planner/tests.rs index d695d13453..ef9c3c585e 100644 --- a/crates/sqlbuild-cursor/src/planner/tests.rs +++ b/crates/sqlbuild-cursor/src/planner/tests.rs @@ -2,6 +2,10 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_cursor_end_clamp.rs"] +mod test_cursor_end_clamp; +#[path = "tests/test_rolling_window.rs"] +mod test_rolling_window; #[path = "tests/test_safety_horizon.rs"] mod test_safety_horizon; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-cursor/src/planner/tests/helpers.rs b/crates/sqlbuild-cursor/src/planner/tests/helpers.rs index b9b6ef1bdd..53ae3202c6 100644 --- a/crates/sqlbuild-cursor/src/planner/tests/helpers.rs +++ b/crates/sqlbuild-cursor/src/planner/tests/helpers.rs @@ -1,6 +1,9 @@ +use crate::algebra::errors::CursorError; use crate::algebra::models::{ - CivilDate, LocalDateTime, Scalar, ScalarValue, Timestamp, ZoneOffset, + CivilDate, DurationSpan, LocalDateTime, Scalar, ScalarValue, Timestamp, ZoneOffset, }; +use crate::algebra::types::ZoneRules; +use crate::planner::tests::test_types::UtcZones; pub(super) fn timestamp(month: u32, day: u32, hour: u32, minute: u32) -> Scalar { Scalar { @@ -24,3 +27,28 @@ pub(super) fn timestamp(month: u32, day: u32, hour: u32, minute: u32) -> Scalar }), } } + +impl ZoneRules for UtcZones { + fn to_utc( + &self, + _: usize, + local: &LocalDateTime, + _: bool, + ) -> Result { + Ok(*local) + } + + fn utc_offset(&self, _: usize, _: &LocalDateTime, _: bool) -> Result, CursorError> { + Ok(Some(0)) + } +} + +pub(super) fn days(days: i64) -> DurationSpan { + DurationSpan { + months: 0, + days, + hours: 0, + minutes: 0, + seconds: 0, + } +} diff --git a/crates/sqlbuild-cursor/src/planner/tests/test_cursor_end_clamp.rs b/crates/sqlbuild-cursor/src/planner/tests/test_cursor_end_clamp.rs new file mode 100644 index 0000000000..41476933ca --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/tests/test_cursor_end_clamp.rs @@ -0,0 +1,43 @@ +use crate::planner::main::cursor_end_clamp::cursor_end_clamp; +use crate::planner::models::CursorEndClamp; +use crate::planner::tests::helpers::timestamp; +use crate::planner::tests::test_types::{CursorEndClampTestCase, UtcZones}; + +#[test] +fn given_window_and_cursor_end_when_clamping_then_window_never_passes_end() { + let test_cases = [ + CursorEndClampTestCase { + description: "window before the end is unchanged", + start_day: 1, + end_day: 5, + expected_clamp: CursorEndClamp::Unchanged, + }, + CursorEndClampTestCase { + description: "window crossing the end is capped", + start_day: 5, + end_day: 15, + expected_clamp: CursorEndClamp::Capped, + }, + CursorEndClampTestCase { + description: "window starting at the end is empty", + start_day: 10, + end_day: 15, + expected_clamp: CursorEndClamp::Empty, + }, + ]; + for test_case in test_cases { + assert_eq!( + cursor_end_clamp( + ( + ×tamp(1, test_case.start_day, 0, 0), + ×tamp(1, test_case.end_day, 0, 0) + ), + ×tamp(1, 10, 0, 0), + &UtcZones, + ), + Ok(test_case.expected_clamp), + "{}", + test_case.description, + ); + } +} diff --git a/crates/sqlbuild-cursor/src/planner/tests/test_rolling_window.rs b/crates/sqlbuild-cursor/src/planner/tests/test_rolling_window.rs new file mode 100644 index 0000000000..a544fc896c --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/tests/test_rolling_window.rs @@ -0,0 +1,40 @@ +use crate::algebra::models::Grain; +use crate::planner::main::rolling_window::rolling_window; +use crate::planner::tests::helpers::{days, timestamp}; +use crate::planner::tests::test_types::RollingWindowTestCase; + +#[test] +fn given_invocation_clock_when_rolling_window_then_window_ends_at_floored_clock() { + let test_cases = [ + RollingWindowTestCase { + description: "day grain floors to midnight and steps back", + days: 2, + grain: Grain::Day, + expected_window: (timestamp(1, 29, 0, 0), timestamp(1, 31, 0, 0)), + }, + RollingWindowTestCase { + description: "hour grain keeps the hour", + days: 1, + grain: Grain::Hour, + expected_window: (timestamp(1, 30, 12, 0), timestamp(1, 31, 12, 0)), + }, + RollingWindowTestCase { + description: "zero span over minute grain is empty", + days: 0, + grain: Grain::Minute, + expected_window: (timestamp(1, 31, 12, 30), timestamp(1, 31, 12, 30)), + }, + ]; + for test_case in test_cases { + assert_eq!( + rolling_window( + ×tamp(1, 31, 12, 30), + &days(test_case.days), + test_case.grain + ), + Ok(test_case.expected_window), + "{}", + test_case.description, + ); + } +} diff --git a/crates/sqlbuild-cursor/src/planner/tests/test_types.rs b/crates/sqlbuild-cursor/src/planner/tests/test_types.rs index f8d56d203a..1e3cccc00a 100644 --- a/crates/sqlbuild-cursor/src/planner/tests/test_types.rs +++ b/crates/sqlbuild-cursor/src/planner/tests/test_types.rs @@ -1,4 +1,5 @@ use crate::algebra::models::{Grain, Scalar}; +use crate::planner::models::CursorEndClamp; pub(super) struct HorizonTestCase { pub(super) description: &'static str, @@ -7,3 +8,19 @@ pub(super) struct HorizonTestCase { pub(super) date_only: bool, pub(super) expected_horizon: Scalar, } + +pub(super) struct UtcZones; + +pub(super) struct RollingWindowTestCase { + pub(super) description: &'static str, + pub(super) days: i64, + pub(super) grain: Grain, + pub(super) expected_window: (Scalar, Scalar), +} + +pub(super) struct CursorEndClampTestCase { + pub(super) description: &'static str, + pub(super) start_day: u32, + pub(super) end_day: u32, + pub(super) expected_clamp: CursorEndClamp, +} diff --git a/crates/sqlbuild-model-config/src/path_defaults/tests/test_select_path_default.rs b/crates/sqlbuild-model-config/src/path_defaults/tests/test_select_path_default.rs index 7431baeb5e..08d632294d 100644 --- a/crates/sqlbuild-model-config/src/path_defaults/tests/test_select_path_default.rs +++ b/crates/sqlbuild-model-config/src/path_defaults/tests/test_select_path_default.rs @@ -45,6 +45,26 @@ fn given_path_default_keys_when_selecting_then_the_python_choice_is_made() { .with_help(CONFLICT_HELP), ), }, + SelectTestCase { + description: "a single glob consumes exactly one segment", + model_path: "models/commerce/eu/partner_feed/staging/orders.sql", + keys: &["commerce/*/staging"], + expected_choice: PathDefaultChoice::Selected(None), + }, + SelectTestCase { + description: "a single glob outranks a recursive glob", + model_path: "models/commerce/partner_feed/staging/orders.sql", + keys: &["commerce/**/staging", "commerce/*/staging"], + expected_choice: PathDefaultChoice::Selected(Some("commerce/*/staging".to_owned())), + }, + SelectTestCase { + description: "a literal prefix outranks a recursive glob on a Windows path", + model_path: "models\\commerce\\partner_feed\\staging\\orders.sql", + keys: &["commerce/**/staging", "commerce/partner_feed/staging"], + expected_choice: PathDefaultChoice::Selected(Some( + "commerce/partner_feed/staging".to_owned(), + )), + }, SelectTestCase { description: "no key matches", model_path: "staging/orders.sql", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index 1436366c08..b7353adbfc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -3,8 +3,11 @@ use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use std::collections::{BTreeMap, BTreeSet}; + +use pyo3::exceptions::{PyOSError, PyTypeError}; +use pyo3::types::{PyDict, PyDictMethods, PySet, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::explicit_references::errors::ReferenceFailure; use sqlbuild_analysis::explicit_references::main::check_explicit_references::check_explicit_references; use sqlbuild_analysis::explicit_references::main::execution_gate_edges::execution_gate_edges; @@ -15,12 +18,14 @@ use sqlbuild_analysis::explicit_references::models::{ ReferenceReport, ReferenceRequest, SqlOwner, }; use sqlbuild_analysis::explicit_references::types::ResourceKey; +use sqlbuild_config::project::main::explicit_references_help::explicit_references_help as references_help; use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_pysource::literal_sql::main::literal_sql_calls::literal_sql_calls; use sqlbuild_pysource::literal_sql::models::FunctionSource; -use crate::bindings::_helpers::analysis::reference_facts::{ProjectFacts, project_facts, ref_keys}; +use crate::bindings::_helpers::analysis::reference_facts::{ProjectFacts, project_facts}; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::types::CompilerDetach; const EXPLICIT_REFERENCES_CONTEXT: &str = "native explicit-reference checks"; @@ -29,48 +34,106 @@ const COMPILE_INPUT_ERROR: &str = "CompileInputError"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const REFERENCE_MODELS_MODULE: &str = "sqlbuild.compiler.references.models"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; +const LOADER_DEFINITION_MODULE: &str = "sqlbuild.python_nodes.main.read_loader_definition"; +const NATIVE_MODULE: &str = "sqlbuild._native"; +const BUILTINS_MODULE: &str = "builtins"; +const VALUE_PLACEHOLDER: &str = "__sqlbuild_python_value_"; +const HARD_CODED_ALLOWED: &str = "hard-coded relation names"; +const NODE_OWNERS: [(&str, &str); 3] = [ + ("task", "task_functions"), + ("asset", "asset_functions"), + ("check", "check_functions"), +]; +const NODE_KIND: &str = "node"; +const HOOK_KIND: &str = "hook"; +const LOADER_KIND: &str = "loader"; +const MODEL_REF_KIND: &str = "model"; +const SOURCE_REF_KIND: &str = "source"; const DIAGNOSTIC_PHASE: &str = "compile"; const DIAGNOSTIC_SEVERITY: &str = "error"; -/// One Python SQL owner as Python collects it: the owner, its path order and literal SQL calls. -type OwnerInput<'py> = (Bound<'py, PyAny>, usize, Vec<(String, String, usize)>); +/// One Python function whose literal SQL the checks read, and the references it owns. +struct PythonSqlOwner<'py> { + label: String, + kind: &'static str, + function: Bound<'py, PyAny>, + relative_path: Bound<'py, PyAny>, + own_refs: Vec, + upstream_loader_by_source: Vec<(String, String)>, +} -/// The project-wide settings of one explicit-reference check. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -struct ReportSettings { - /// Whether the project keeps old-name views unless a model disables them. - retains_old_names: bool, - enforce_explicit: bool, - explicit_help: String, - /// The text Python substitutes for f-string values in literal SQL. - placeholder: String, +impl<'py> PythonSqlOwner<'py> { + fn new(label: String, kind: &'static str, node: &Bound<'py, PyAny>) -> PyResult { + Ok(Self { + label, + kind, + function: node.getattr("function")?, + relative_path: node.getattr("relative_path")?, + own_refs: Vec::new(), + upstream_loader_by_source: Vec::new(), + }) + } } -/// Python's audit-gate, hook-read and Python-SQL checks as a `PythonSqlReferenceReport`. +/// Return P005, P007 and P008 errors and unmatched Python-SQL relations; raise P001. #[pyfunction] +#[pyo3(signature = (*, project, discovered_inputs, enforce_explicit))] fn explicit_reference_report<'py>( py: Python<'py>, project: &Bound<'py, PyAny>, - settings: ReportSettings, - owners: Vec>, + discovered_inputs: &Bound<'py, PyAny>, + enforce_explicit: bool, ) -> PyResult> { - let facts: ProjectFacts = project_facts(project, settings.retains_old_names)?; + let owners: Vec> = if enforce_explicit { + python_sql_owners(project, discovered_inputs)? + } else { + Vec::new() + }; + let path_order: Bound<'py, PyDict> = PyDict::new(py); + let unique_paths: Bound<'py, PySet> = + PySet::new(py, owners.iter().map(|owner| owner.relative_path.clone()))?; + for (index, path) in imported(py, BUILTINS_MODULE, "sorted")? + .call1((unique_paths,))? + .try_iter()? + .enumerate() + { + path_order.set_item(path?, index)?; + } + let retains_old_names: bool = !discovered_inputs + .getattr("project_config")? + .getattr("migrations")? + .getattr("old_name_views")? + .is_none(); + let facts: ProjectFacts = project_facts(project, retains_old_names)?; + let mut sql_owners: Vec = Vec::with_capacity(owners.len()); + for owner in &owners { + let order: usize = path_order + .as_any() + .get_item(&owner.relative_path)? + .extract()?; + sql_owners.push(sql_owner( + owner, + order, + literal_sql_calls_of(&owner.function)?, + )); + } let request = ReferenceRequest { - enforce_explicit: settings.enforce_explicit, + enforce_explicit, dialect: project.getattr("sql_analysis_dialect")?.extract()?, - explicit_help: settings.explicit_help, - placeholder: settings.placeholder, + explicit_help: if enforce_explicit { + references_help(HARD_CODED_ALLOWED) + } else { + String::new() + }, + placeholder: VALUE_PLACEHOLDER.to_owned(), resources: facts.resources, models: facts.models, sources: facts.sources, seeds: facts.seeds, audits: facts.audits, hooks: facts.hooks, - owners: owners - .iter() - .map(|(owner, path_order, literals)| sql_owner(owner, *path_order, literals)) - .collect::>()?, + owners: sql_owners, }; match py .compiler_detach(|| Ok(check_explicit_references(&request))) @@ -90,39 +153,168 @@ fn explicit_reference_report<'py>( } } +/// Every task, asset, check, loader and hook function, as Python discovery lists them. +fn python_sql_owners<'py>( + project: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = project.py(); + let mut owners: Vec> = Vec::new(); + for (label, attribute) in NODE_OWNERS { + for node in discovered_inputs.getattr(attribute)?.try_iter()? { + let node: Bound<'py, PyAny> = node?; + let name: String = node.getattr("name")?.extract()?; + owners.push(PythonSqlOwner::new( + format!("{label}:{name}"), + NODE_KIND, + &node, + )?); + } + } + owners.extend(loader_owners(project, discovered_inputs)?); + let hook_names = imported(py, NATIVE_MODULE, "model_python_hook_names")?; + let mut attached: BTreeMap> = BTreeMap::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("model", &model)?; + let model_name: String = model.getattr("name")?.extract()?; + for hook_name in hook_names.call((), Some(&keywords))?.try_iter()? { + attached + .entry(hook_name?.extract()?) + .or_default() + .insert(model_name.clone()); + } + } + for hook in project.getattr("hook_functions")?.try_iter()? { + let hook: Bound<'py, PyAny> = hook?; + let name: String = hook.getattr("name")?.extract()?; + let mut owner = PythonSqlOwner::new(format!("hook:{name}"), HOOK_KIND, &hook)?; + owner.own_refs = attached + .get(&name) + .into_iter() + .flatten() + .map(|model| (MODEL_REF_KIND.to_owned(), model.clone())) + .collect(); + owners.push(owner); + } + Ok(owners) +} + +/// Each loader function, owning its sources and naming the loaders it depends on. +fn loader_owners<'py>( + project: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = project.py(); + let mut sources_by_loader: BTreeMap> = BTreeMap::new(); + for source in project.getattr("sources")?.try_iter()? { + let source: Bound<'py, PyAny> = source?; + let loader: Option = source + .getattr("source_entry")? + .getattr("loader")? + .extract()?; + if let Some(loader) = loader { + sources_by_loader + .entry(loader) + .or_default() + .insert(source.getattr("name")?.extract()?); + } + } + let read_definition = imported(py, LOADER_DEFINITION_MODULE, "read_loader_definition")?; + let mut owners: Vec> = Vec::new(); + for loader in discovered_inputs.getattr("loader_functions")?.try_iter()? { + let loader: Bound<'py, PyAny> = loader?; + let name: String = loader.getattr("name")?.extract()?; + let mut upstream: Vec<(String, String)> = Vec::new(); + for dependency in loader.getattr("depends_on")?.try_iter()? { + let dependency: Bound<'py, PyAny> = dependency?; + if !dependency.is_callable() { + continue; + } + let definition: Bound<'py, PyAny> = read_definition.call1((&dependency,))?; + if definition.is_none() { + continue; + } + let definition_name: String = definition.getattr("name")?.extract()?; + let shown: String = match dependency.getattr_opt("__name__")? { + Some(shown) => shown.extract()?, + None => definition_name.clone(), + }; + for source_name in sources_by_loader + .get(&definition_name) + .into_iter() + .flatten() + { + upstream.retain(|(known, _)| known != source_name); + upstream.push((source_name.clone(), shown.clone())); + } + } + let mut owner = PythonSqlOwner::new(format!("loader:{name}"), LOADER_KIND, &loader)?; + owner.own_refs = sources_by_loader + .get(&name) + .into_iter() + .flatten() + .map(|source| (SOURCE_REF_KIND.to_owned(), source.clone())) + .collect(); + owner.upstream_loader_by_source = upstream; + owners.push(owner); + } + Ok(owners) +} + +/// The literal SQL calls of a function's dedented source, or none when its source is unreadable. +fn literal_sql_calls_of(function: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = function.py(); + let (lines, start_line): (Bound<'_, PyAny>, usize) = + match imported(py, "inspect", "getsourcelines")?.call1((function,)) { + Ok(found) => found.extract()?, + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + return Ok(Vec::new()); + } + Err(error) => return Err(error), + }; + let joined: Bound<'_, PyAny> = PyString::new(py, "").call_method1("join", (lines,))?; + let source: String = imported(py, "textwrap", "dedent")? + .call1((joined,))? + .extract()?; + Ok(python_literal_sql_calls( + py, + &source, + start_line, + VALUE_PLACEHOLDER, + )) +} + fn sql_owner( - owner: &Bound<'_, PyAny>, + owner: &PythonSqlOwner<'_>, path_order: usize, - literals: &[(String, String, usize)], -) -> PyResult { - Ok(SqlOwner { - label: owner.getattr("label")?.extract()?, - kind: owner.getattr("kind")?.str()?.extract()?, + literals: Vec<(String, String, usize)>, +) -> SqlOwner { + SqlOwner { + label: owner.label.clone(), + kind: owner.kind.to_owned(), path_order, - own_refs: ref_keys(&owner.getattr("own_refs")?)?, - upstream_loaders: owner - .getattr("upstream_loader_by_source")? - .call_method0("items")? - .try_iter()? - .map(|item| item?.extract()) - .collect::>()?, + own_refs: owner.own_refs.clone(), + upstream_loaders: owner.upstream_loader_by_source.clone(), literals: literals - .iter() - .map(|(method, sql, line)| LiteralSql { - method: method.clone(), - sql: sql.clone(), - line: *line, - }) + .into_iter() + .map(|(method, sql, line)| LiteralSql { method, sql, line }) .collect(), - }) + } } /// The report's diagnostics and unmatched relations as Python's compile models. fn python_report<'py>( py: Python<'py>, report: &ReferenceReport, - owners: &[OwnerInput<'py>], + owners: &[PythonSqlOwner<'py>], ) -> PyResult> { + let owner_kind: Bound<'py, PyAny> = + imported(py, REFERENCE_TYPES_MODULE, "HardCodedRelationOwnerKind")?; let compile_models = py.import(COMPILE_MODELS_MODULE)?; let reference_models = py.import(REFERENCE_MODELS_MODULE)?; let source_location: Bound<'py, PyAny> = py @@ -146,7 +338,7 @@ fn python_report<'py>( match row.owner { Some(owner) => { let location: Bound<'py, PyDict> = PyDict::new(py); - location.set_item("path", owners[owner].0.getattr("relative_path")?)?; + location.set_item("path", &owners[owner].relative_path)?; location.set_item("line", row.line)?; location.set_item("column", 1)?; fields.set_item("location", source_location.call((), Some(&location))?)?; @@ -160,15 +352,15 @@ fn python_report<'py>( let relation_name: Bound<'py, PyAny> = reference_models.getattr("RelationName")?; let mut unmatched: Vec> = Vec::with_capacity(report.unmatched.len()); for row in &report.unmatched { - let owner: &Bound<'py, PyAny> = &owners[row.owner].0; + let owner: &PythonSqlOwner<'py> = &owners[row.owner]; let relation: Bound<'py, PyDict> = PyDict::new(py); relation.set_item("name", &row.relation.name)?; relation.set_item("schema", &row.relation.schema)?; relation.set_item("database", &row.relation.database)?; let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item("owner_label", owner.getattr("label")?)?; - fields.set_item("owner_kind", owner.getattr("kind")?)?; - fields.set_item("relative_path", owner.getattr("relative_path")?)?; + fields.set_item("owner_label", &owner.label)?; + fields.set_item("owner_kind", owner_kind.call1((owner.kind,))?)?; + fields.set_item("relative_path", &owner.relative_path)?; fields.set_item("line", row.line)?; fields.set_item("method", &row.method)?; fields.set_item("relation", relation_name.call((), Some(&relation))?)?; @@ -183,7 +375,6 @@ fn python_report<'py>( } /// `(method, sql, line)` for each literal SQL call to the context in a dedented function source. -#[pyfunction] fn python_literal_sql_calls( py: Python<'_>, source: &str, @@ -278,7 +469,6 @@ fn hook_row(edge: HookEdge) -> HookRow { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(explicit_reference_report, module)?)?; - module.add_function(wrap_pyfunction!(python_literal_sql_calls, module)?)?; module.add_function(wrap_pyfunction!(hard_coded_relation_remedy, module)?)?; module.add_function(wrap_pyfunction!(old_name_reference_message, module)?)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 7dea30fd8a..911eff0bb4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -11,10 +11,10 @@ use crate::bindings::_helpers::analysis_session::profile_inputs::{ use crate::bindings::_helpers::analysis_session::stage_functions::family_rows; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::analysis_cache::record_analysis_cache_metrics; const NATIVE_MODULE: &str = "sqlbuild._native"; const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; -const ANALYSIS_CACHE_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.cache"; const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const CODE_IDENTITY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiled_code_identity"; @@ -260,15 +260,7 @@ fn record_cache(python: &[Bound<'_, PyAny>], session: &Bound<'_, PyAny>) -> PyRe /// Record the analysis cache entries a native session read, missed and bypassed. fn record_metrics(py: Python<'_>, (hits, misses, bypasses): (usize, usize, usize)) -> PyResult<()> { - let arguments: Bound<'_, PyDict> = PyDict::new(py); - arguments.set_item("batch_hits", 0)?; - arguments.set_item("entry_hits", hits)?; - arguments.set_item("misses", misses)?; - arguments.set_item("bypasses", bypasses)?; - py.import(ANALYSIS_CACHE_MODULE)? - .getattr("record_analysis_cache_metrics")? - .call((), Some(&arguments))?; - Ok(()) + record_analysis_cache_metrics(py, (0, hits, misses, bypasses)) } /// Each model's native request row, in analysis order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index c6c24751b6..c7b6027704 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -19,8 +19,8 @@ const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; -const AUDIT_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._audit"; -const COST_CONTEXT_MODULE: &str = "sqlbuild.cost.classes.cost_context"; +const AUDIT_PLAN_MODULE: &str = "sqlbuild._native"; +const COST_CONTEXT_MODULE: &str = "sqlbuild._native"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const PROJECTION_MODULE: &str = "sqlbuild.executor.auditing.main._current_audit_result_projection"; const EXECUTION_COMMAND: &str = "sqb audit"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs index 6b169c8998..3d904ab50e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs @@ -11,10 +11,9 @@ use sqlbuild_model_config::header_metadata::models::ParsedAudit; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::model_config::contract_classes::contract_classes; use crate::bindings::_helpers::model_config::header_metadata::{ContractClasses, audit_tuple}; -const CONTRACT_CLASSES_MODULE: &str = - "sqlbuild.compiler.model_config.main._native_contract_classes"; const IDENTITY_ERRORS_MODULE: &str = "sqlbuild.compiler.resource_names.exceptions"; /// A resource identity error as `ResourceIdentityError`, anything else as `error_class(message)`. @@ -54,11 +53,7 @@ fn parse_audit_instances<'py>( Ok(parsed) => parsed, Err(error) => return Err(raised(py, error, &error_class)?), }; - let classes: ContractClasses<'py> = py - .import(CONTRACT_CLASSES_MODULE)? - .getattr("native_contract_classes")? - .call0()? - .extract()?; + let classes: ContractClasses<'py> = contract_classes(py)?.extract()?; audit_tuple(py, &classes, &parsed) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/expected_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/expected_errors.rs new file mode 100644 index 0000000000..275cde1b9c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/expected_errors.rs @@ -0,0 +1,38 @@ +//! The stable code, message and help of an expected exception-like object. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::PyString; +use pyo3::{pyfunction, wrap_pyfunction}; + +/// Return a stable code from an expected exception-like object. +#[pyfunction] +#[pyo3(signature = (*, error, fallback_code))] +fn error_code(error: &Bound<'_, PyAny>, fallback_code: &Bound<'_, PyAny>) -> PyResult { + match error.getattr_opt("code")? { + Some(code) => Ok(code.str()?.to_string()), + None => Ok(fallback_code.str()?.to_string()), + } +} + +/// Return expected-error help text when present. +#[pyfunction] +fn error_help<'py>(error: &Bound<'py, PyAny>) -> PyResult>> { + Ok(error + .getattr_opt("help")? + .filter(|help| help.is_instance_of::())) +} + +/// Return a stable message from an expected exception-like object. +#[pyfunction] +fn error_message(error: &Bound<'_, PyAny>) -> PyResult { + match error.getattr_opt("message")? { + Some(message) => Ok(message.str()?.to_string()), + None => Ok(error.str()?.to_string()), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(error_code, module)?)?; + module.add_function(wrap_pyfunction!(error_help, module)?)?; + module.add_function(wrap_pyfunction!(error_message, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs index 7c0e808fc6..56be30d4df 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs @@ -1,5 +1,6 @@ mod allocator; pub(crate) mod config_values; +pub(crate) mod expected_errors; pub(crate) mod keyword_request; pub(crate) mod oracles; pub(crate) mod panics; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 593374b7c0..9069c440bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::build_command::planning::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; -const TIMING_TRACKER_MODULE: &str = "sqlbuild.diagnostics.classes.build_phase_timing_tracker"; +const TIMING_TRACKER_MODULE: &str = "sqlbuild._native"; const RESOURCE_REPORTING_MODULE: &str = "sqlbuild.diagnostics.main.process_resource_reporting"; const DEADLINE_MODULE: &str = "sqlbuild.runtime.execution_limits.main.execution_deadline_scope"; const PROVIDER_SESSION_MODULE: &str = "sqlbuild.provider.main.session"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index 0bb5ab41d2..1165ddcea3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -6,7 +6,8 @@ use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyCFunction, PyDict, PyFrozenSet, PyString, PyTuple, PyTupleMethods}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_core::build_command::constants::EXECUTION_LIMIT_BUILD_NOTE; use sqlbuild_core::build_command::errors::BuildCommandError; @@ -15,14 +16,11 @@ use crate::bindings::_helpers::build_command::policies::build_command_error; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const CONSTANTS_MODULE: &str = "sqlbuild.cli.commands.constants"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const CLONE_MODELS: &str = "sqlbuild.executor.clone.models"; const CLONE_TYPES: &str = "sqlbuild.executor.clone.types"; const SPEC_TYPES: &str = "sqlbuild.spec.contracts.types"; -const CLONE_BOUNDARY_MODULE: &str = "sqlbuild.compiler.planner.main.clone.resolve_clone_boundary"; -const VIEW_CHAIN_MODULE: &str = "sqlbuild.compiler.planner.main.clone.resolve_skipped_view_chain"; const TRANSIENT_MODELS_MODULE: &str = "sqlbuild.executor.clone.main._build_destination_transient_models"; const RETENTION_REQUESTS_MODULE: &str = "sqlbuild.executor.clone.main._build_retention_requests"; @@ -164,14 +162,10 @@ fn defer_clone_selection(scope: &Bound<'_, PyAny>) -> PyResult<(Vec, Vec let is_view: Bound<'_, PyAny> = membership(py, &keys, |key| recreated_over_clones(scope, key))?; let callbacks: (&Bound<'_, PyAny>, &Bound<'_, PyAny>) = (&is_clonable, &is_view); Ok(( + traversed_names(scope, (NATIVE_MODULE, "resolve_clone_boundary"), callbacks)?, traversed_names( scope, - (CLONE_BOUNDARY_MODULE, "resolve_clone_boundary"), - callbacks, - )?, - traversed_names( - scope, - (VIEW_CHAIN_MODULE, "resolve_skipped_view_chain"), + (NATIVE_MODULE, "resolve_skipped_view_chain"), callbacks, )?, selected_executable_model_count(scope)?, @@ -639,7 +633,7 @@ pub(crate) fn run_defer_clone_boundary_prephase( ("limits", invocation.getattr("execution_limits")?), ( "refusal_note", - imported(py, CONSTANTS_MODULE, "EXECUTION_LIMIT_BUILD_NOTE")?, + EXECUTION_LIMIT_BUILD_NOTE.into_bound_py_any(py)?, ), ], )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 865ea135f9..a35915276a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -22,13 +22,12 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const CURSOR_OVERRIDES_MODULE: &str = - "sqlbuild.compiler.planner.main.cursor_window._typed_cursor_overrides"; +const CURSOR_OVERRIDES_MODULE: &str = "sqlbuild._native"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const PIPELINE_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const FRESHNESS_MODULE: &str = "sqlbuild.cli.commands._helpers.freshness.source_freshness"; -const COST_CONTEXT_MODULE: &str = "sqlbuild.cost.classes.cost_context"; +const COST_CONTEXT_MODULE: &str = "sqlbuild._native"; const GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const CHECKS_MODULE: &str = "sqlbuild.executor.python_nodes.main._checks"; const EXECUTOR_PYTHON_MODELS: &str = "sqlbuild.executor.python_nodes.models"; @@ -179,7 +178,7 @@ fn cursors<'py>( let mut overrides: Bound<'py, PyAny> = plan.getattr("cursor_overrides")?; if !overrides.is_truthy()? { overrides = call_with( - &imported(py, CURSOR_OVERRIDES_MODULE, "typed_cursor_overrides")?, + &imported(py, CURSOR_OVERRIDES_MODULE, "type_cursor_overrides")?, &[( "cursor_overrides", context.request.getattr("cursor_overrides")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index 64be5880e4..ddc33b18dd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -1,6 +1,10 @@ //! Compile the build plan, show it, gate it and write the compile target. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use sqlbuild_core::build_command::constants::{ + EXECUTION_LIMIT_BUILD_NOTE, MISSING_ORIGIN_BUILD_HELP, RETENTION_DECREASE_BUILD_HELP, + SNAPSHOT_FULL_REFRESH_BUILD_HELP, TABLE_TYPE_DOWNGRADE_BUILD_HELP, +}; use sqlbuild_core::build_command::models::PlanSafetyGates; use crate::bindings::_helpers::build_command::defer_clone::run_defer_clone_boundary_prephase; @@ -13,7 +17,6 @@ use crate::bindings::_helpers::cli::progress_text::with_verbose_inspection; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const CONSTANTS_MODULE: &str = "sqlbuild.cli.commands.constants"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; @@ -148,10 +151,6 @@ pub(crate) fn write_plan_text( emit(&stream, &format!("\n{plan_text}\n\n")) } -fn constant(py: Python<'_>, name: &str) -> PyResult { - imported(py, CONSTANTS_MODULE, name)?.extract() -} - /// Apply every plan safety gate with the build's allow flags. pub(crate) fn enforce_build_plan_policies( context: &BuildContext<'_>, @@ -163,11 +162,11 @@ pub(crate) fn enforce_build_plan_policies( allow_snapshot_full_refresh: context.request_flag("allow_snapshot_full_refresh")?, allow_table_type_downgrade: context.request_flag("allow_table_type_downgrade")?, allow_retention_decrease: context.request_flag("allow_retention_decrease")?, - missing_origin_help: constant(py, "MISSING_ORIGIN_BUILD_HELP")?, - snapshot_full_refresh_help: constant(py, "SNAPSHOT_FULL_REFRESH_BUILD_HELP")?, - table_type_downgrade_help: constant(py, "TABLE_TYPE_DOWNGRADE_BUILD_HELP")?, - retention_decrease_help: constant(py, "RETENTION_DECREASE_BUILD_HELP")?, - execution_limit_note: constant(py, "EXECUTION_LIMIT_BUILD_NOTE")?, + missing_origin_help: MISSING_ORIGIN_BUILD_HELP.to_owned(), + snapshot_full_refresh_help: SNAPSHOT_FULL_REFRESH_BUILD_HELP.to_owned(), + table_type_downgrade_help: TABLE_TYPE_DOWNGRADE_BUILD_HELP.to_owned(), + retention_decrease_help: RETENTION_DECREASE_BUILD_HELP.to_owned(), + execution_limit_note: EXECUTION_LIMIT_BUILD_NOTE.to_owned(), }; let sys: Bound<'_, PyAny> = py.import("sys")?.into_any(); enforce_plan_policies( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs index e0efa09893..10bc878014 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs @@ -6,4 +6,5 @@ pub(crate) mod plan_facts; pub(crate) mod registration; pub(crate) mod result_text; pub(crate) mod terminal; +pub(crate) mod transient_lines; pub(crate) mod transient_status; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs index 27abb010e4..4330892110 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs @@ -3,7 +3,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::build_progress::{ - audit_progress, build_callbacks, footer, nested_progress, terminal, transient_status, + audit_progress, build_callbacks, footer, nested_progress, terminal, transient_lines, + transient_status, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -12,5 +13,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { footer::register(module)?; nested_progress::register(module)?; terminal::register(module)?; + transient_lines::register(module)?; transient_status::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs index c0ddf2193d..ebf0e651f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs @@ -4,10 +4,10 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::build_progress::transient_lines::shared_coordinator; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -const COLUMNS_MODULE: &str = "sqlbuild.presentation.main.terminal_columns"; -const LINES_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; +const DEFAULT_TERMINAL_SIZE: (i64, i64) = (120, 24); const SPINNER_JOIN_SECONDS: f64 = 0.2; const EXIT_JOIN_SECONDS: f64 = 1.0; const SPINNER_THREAD_NAME: &str = "sqlbuild-progress-spinner"; @@ -21,17 +21,20 @@ pub(crate) const SPINNER_FRAMES: [&str; 10] = [ "\u{2807}", "\u{280f}", ]; -/// The current terminal width. +/// The current terminal width, `shutil.get_terminal_size(fallback=(120, 24)).columns`. pub(crate) fn terminal_columns(py: Python<'_>) -> PyResult { - let columns: i64 = imported(py, COLUMNS_MODULE, "terminal_columns")? - .call0()? + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("fallback", DEFAULT_TERMINAL_SIZE)?; + let columns: i64 = imported(py, "shutil", "get_terminal_size")? + .call((), Some(&options))? + .getattr("columns")? .extract()?; Ok(usize::try_from(columns).unwrap_or(0)) } /// The process-wide coordinator of the one live transient line. pub(crate) fn shared_lines(py: Python<'_>) -> PyResult> { - imported(py, LINES_MODULE, "shared_transient_line_coordinator")?.call0() + Ok(shared_coordinator(py)?.into_any()) } /// Whether `stream` reports a terminal. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs new file mode 100644 index 0000000000..e746a615ab --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs @@ -0,0 +1,141 @@ +//! The coordinator of one live transient terminal line and persistent writes. + +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; + +use crate::bindings::_helpers::build_progress::terminal::tty_flag; +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +/// The stream a transient line is drawn on and the renderer that owns it. +type ActiveLine = (Py, Py); + +static SHARED_COORDINATOR: PyOnceLock> = PyOnceLock::new(); + +/// Keep persistent terminal lines from landing on a live spinner or status line. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct TransientLineCoordinator { + /// The re-entrant lock every writer holds while it draws. + #[pyo3(get)] + lock: Py, + active: Mutex>, +} + +impl TransientLineCoordinator { + fn active(&self) -> MutexGuard<'_, Option> { + self.active.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Run `body` while holding the coordinator lock. + fn locked(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { + let lock: &Bound<'_, PyAny> = self.lock.bind(py); + lock.call_method0("acquire")?; + let result: PyResult = body(); + lock.call_method0("release")?; + result + } + + /// The owner whose line a write to `stream` would land on. + fn interrupted_owner<'py>( + &self, + stream: &Bound<'py, PyAny>, + ) -> PyResult>> { + let py: Python<'py> = stream.py(); + let Some((owner_stream, owner)) = self + .active() + .as_ref() + .map(|(owner_stream, owner)| (owner_stream.bind(py).clone(), owner.bind(py).clone())) + else { + return Ok(None); + }; + if owner_stream.is(stream) || (tty_flag(&owner_stream)? && tty_flag(stream)?) { + return Ok(Some(owner)); + } + Ok(None) + } +} + +#[pymethods] +impl TransientLineCoordinator { + #[new] + fn new(py: Python<'_>) -> PyResult { + Ok(Self { + lock: imported(py, "threading", "RLock")?.call0()?.unbind(), + active: Mutex::new(None), + }) + } + + /// Record the owner of the transient line currently drawn on a terminal stream. + #[pyo3(signature = (*, stream, owner))] + fn claim(&self, stream: &Bound<'_, PyAny>, owner: &Bound<'_, PyAny>) -> PyResult<()> { + self.locked(stream.py(), || { + *self.active() = Some((stream.clone().unbind(), owner.clone().unbind())); + Ok(()) + }) + } + + /// Forget a transient owner once its line is no longer drawn. + #[pyo3(signature = (*, owner))] + fn release(&self, owner: &Bound<'_, PyAny>) -> PyResult<()> { + self.locked(owner.py(), || { + let mut active = self.active(); + if active + .as_ref() + .is_some_and(|(_, current)| current.is(owner)) + { + *active = None; + } + Ok(()) + }) + } + + /// Erase and forget any live transient line, so nothing redraws it afterwards. + fn clear_active(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || { + let taken: Option = self.active().take(); + if let Some((_, owner)) = taken { + owner.bind(py).call_method0("clear_transient_line")?; + } + Ok(()) + }) + } + + /// Write lasting text, clearing and then redrawing any live transient line. + #[pyo3(signature = (*, stream, text))] + fn write_persistent(&self, stream: &Bound<'_, PyAny>, text: &Bound<'_, PyAny>) -> PyResult<()> { + self.locked(stream.py(), || { + let owner: Option> = self.interrupted_owner(stream)?; + if let Some(owner) = &owner { + owner.call_method0("clear_transient_line")?; + } + stream.call_method1("write", (text,))?; + stream.call_method0("flush")?; + if let Some(owner) = &owner { + owner.call_method0("redraw_transient_line")?; + } + Ok(()) + }) + } +} + +/// The coordinator shared by every terminal writer in this process. +pub(crate) fn shared_coordinator(py: Python<'_>) -> PyResult> { + SHARED_COORDINATOR + .get_or_try_init(py, || Py::new(py, TransientLineCoordinator::new(py)?)) + .map(|coordinator| coordinator.bind(py).clone()) +} + +/// Return the coordinator shared by every terminal writer in this process. +#[pyfunction] +fn shared_transient_line_coordinator( + py: Python<'_>, +) -> PyResult> { + shared_coordinator(py) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(shared_transient_line_coordinator, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs index 6c69245ecd..a315e0f06a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs @@ -1,7 +1,8 @@ //! Whole per-file SQL-test scan results kept as one kind of the shared native store. use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyclass, pymethods}; use sqlbuild_cache::digest::main::content_digest::content_digest; use sqlbuild_cache::digest::main::hex_digest::hex_digest; @@ -12,6 +13,11 @@ use std::sync::Mutex; use std::sync::atomic::{AtomicUsize, Ordering}; use crate::bindings::_helpers::cache::native_store::{open_store, save_store}; +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const CODE_IDENTITY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiled_code_identity"; +const METRIC_MODULE: &str = "sqlbuild.compiler.profiling.main._metric"; +const SCAN_CACHE_LOGGER: &str = "sqlbuild.compile"; /// The native store kind holding SQL-test scan results. const SQL_TEST_SCAN_STORE_KIND: &str = "sql-test-scans"; @@ -111,8 +117,81 @@ impl SqlTestScanStore { } } +/// The compile cache folder's scan store, opened on first use and saved once. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct SqlTestScanCache { + state: Mutex<(Option, Option>)>, +} + +#[pymethods] +impl SqlTestScanCache { + #[new] + #[pyo3(signature = (*, cache_dir))] + fn new(cache_dir: Option) -> Self { + Self { + state: Mutex::new((cache_dir, None)), + } + } + + /// The native store this compile reads and writes, or None when it has none. + fn opened(&self, py: Python<'_>) -> PyResult>> { + let mut state = self.state.lock().map_err(poisoned)?; + let (cache_dir, store) = &mut *state; + let Some(directory) = cache_dir.as_ref().filter(|_| store.is_none()) else { + return Ok(store.as_ref().map(|store| store.clone_ref(py))); + }; + let identity: String = imported(py, CODE_IDENTITY_MODULE, "compiled_code_identity")? + .call0()? + .extract()?; + let (opened, error) = SqlTestScanStore::open_in(py, directory.clone(), &identity); + *store = opened.map(|opened| Py::new(py, opened)).transpose()?; + if let Some(error) = error { + debug(py, "SQL test scan store not loaded: %s", &error)?; + *cache_dir = None; + } + Ok(store.as_ref().map(|store| store.clone_ref(py))) + } + + /// Persist results stored by this compile and report hit and miss counts. + fn save(&self, py: Python<'_>) -> PyResult<()> { + let store: Option> = self + .state + .lock() + .map_err(poisoned)? + .1 + .as_ref() + .map(|store| store.clone_ref(py)); + let (hits, misses) = store.as_ref().map_or((0, 0), |store| store.get().counts()); + metric(py, "sql_test_scan_cache_hits", hits)?; + metric(py, "sql_test_scan_cache_misses", misses)?; + let Some(store) = store else { + return Ok(()); + }; + if let (_, Some(error)) = store.get().save(py)? { + debug(py, "SQL test scan store not saved: %s", &error)?; + } + Ok(()) + } +} + +fn metric(py: Python<'_>, name: &str, value: usize) -> PyResult<()> { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("metric", name)?; + keywords.set_item("value", value)?; + imported(py, METRIC_MODULE, "record_compile_metric")?.call((), Some(&keywords))?; + Ok(()) +} + +fn debug(py: Python<'_>, message: &str, error: &str) -> PyResult<()> { + imported(py, "logging", "getLogger")? + .call1((SCAN_CACHE_LOGGER,))? + .call_method1("debug", (message, error))?; + Ok(()) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; + module.add_class::()?; module.add( "SQL_TEST_SCAN_STORE_FILE_NAME", SQL_TEST_SCAN_STORE_FILE_NAME, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index b40ea8a85c..aec20540f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -31,7 +31,7 @@ const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; -const RELATION_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._relation_plan"; +const RELATION_PLAN_MODULE: &str = "sqlbuild._native"; const SELECTOR_PARSE_MODULE: &str = "sqlbuild._native"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index b5e2c3f20d..483a944ba9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -33,7 +33,7 @@ const EXTERNAL_SCOPE_MODULE: &str = const INVOCATION_CONTEXT_MODULE: &str = "sqlbuild.runtime.output_capture.main.invocation_context"; const OUTPUT_CAPTURE_ERRORS_MODULE: &str = "sqlbuild.runtime.output_capture.exceptions"; const DIAGNOSTICS_MODULE: &str = "sqlbuild.diagnostics.main.diagnostics_context"; -const TRANSIENT_LINE_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; +const TRANSIENT_LINE_MODULE: &str = "sqlbuild._native"; const SKILLS_MODULE: &str = "sqlbuild._native"; const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; const LINT_ERRORS_MODULE: &str = "sqlbuild.lint.exceptions"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs index a74d0e08c5..9348de724d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs @@ -22,7 +22,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; const CONTEXT_ATTRIBUTE: &str = "_native_progress_projector_context"; const CONTEXT_NAME: &str = "sqlbuild_native_progress_projector"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COORDINATOR_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; +const COORDINATOR_MODULE: &str = "sqlbuild._native"; const COORDINATOR_FUNCTION: &str = "shared_transient_line_coordinator"; const CLAIM_OPTIONS: (&[&str], &[&str]) = ( &["resource_name"], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index b49ba9c7b5..aa0308a836 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -8,10 +8,10 @@ use crate::bindings::_helpers::cli::{ }; use crate::bindings::_helpers::{ audit_command, build_command, build_progress, check, clone, command_runtime, contract_command, - cost_command, cost_report, dbt_command, dbt_events, dbt_interop, debug_command, diff_report, - format_command, freshness, init_command, janitor, load, plan_command, presentation_text, - query_command, result_output, scenario_command, seed_command, sql_test_command, - workspace_commands, + cost_command, cost_report, cost_telemetry, dbt_command, dbt_events, dbt_interop, debug_command, + diff_report, format_command, freshness, init_command, janitor, load, plan_command, + presentation_text, query_command, result_output, scenario_command, seed_command, + sql_test_command, workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -30,6 +30,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { load::registration::register(module)?; result_output::registration::register(module)?; cost_report::registration::register(module)?; + cost_telemetry::registration::register(module)?; debug_command::registration::register(module)?; dbt_events::registration::register(module)?; dbt_interop::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs index 1677450c25..98bad823cb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs @@ -11,7 +11,7 @@ use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; -const CLONE_PLANNING_MODULE: &str = "sqlbuild.compiler.planner.main.clone._clone"; +const CLONE_PLANNING_MODULE: &str = "sqlbuild._native"; /// The `ClonePipelineResult` fields after the two projects, in `run_clone_planning` order. const PLANNED_FIELDS: [&str; 8] = [ "clone_plan", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs index ae3c52c241..53167885f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs @@ -7,11 +7,13 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_config::command_connection::constants::COMMAND_WAREHOUSE_GROUPS; + use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; -const COMMAND_CONSTANTS: &str = "sqlbuild.cli.commands.constants"; +const WAREHOUSE_GROUP: (&str, &str) = ("sqlbuild.spec.contracts.types", "WarehouseGroup"); const COMMAND_TYPES: &str = "sqlbuild.cli.commands.types"; const CONTEXT_ATTRIBUTE: &str = "_command_warehouse_scope_context"; const CONTEXT_NAME: &str = "sqlbuild_command_warehouse_scope"; @@ -65,10 +67,18 @@ pub(crate) fn command_warehouse_scope<'py>( let group: Bound<'py, PyAny> = if command.is_none() { py.None().into_bound(py) } else { - imported(py, COMMAND_CONSTANTS, "COMMAND_WAREHOUSE_GROUPS")?.call_method1( - "get", - (imported(py, COMMAND_TYPES, "CliCommand")?.call1((command,))?,), - )? + let name: String = imported(py, COMMAND_TYPES, "CliCommand")? + .call1((command,))? + .str()? + .to_string(); + match COMMAND_WAREHOUSE_GROUPS + .iter() + .find(|(candidate, _)| *candidate == name) + .and_then(|(_, group)| *group) + { + Some(group) => imported(py, WAREHOUSE_GROUP.0, WAREHOUSE_GROUP.1)?.call1((group,))?, + None => py.None().into_bound(py), + } }; let vars: Bound<'py, PyAny> = match cli_vars.filter(|cli_vars| !cli_vars.is_none()) { Some(cli_vars) => py.get_type::().call1((cli_vars,))?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs index f3769dc70d..82f8bd7b0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -11,7 +11,7 @@ use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; use crate::bindings::_helpers::scopes::scope_validation::validate_scope_index; const NATIVE_MODULE: &str = "sqlbuild._native"; -const SCAN_CACHE_MODULE: &str = "sqlbuild.compiler.compile.classes.sql_test_scan_cache"; +const SCAN_CACHE_MODULE: &str = "sqlbuild._native"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs index 14b2053410..2bca63eb99 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs @@ -7,11 +7,11 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::compile_attachment::storage_policies::inherited_storage; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::compiled_project::loop_classes::table_backed; +use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; /// Capture the project inputs every model's native config build and validation read. @@ -49,10 +49,7 @@ pub(crate) fn native_model_config_session<'py>( project_config.getattr("path_defaults")?, hook_entries, ), - ( - inputs.getattr("effective_vars")?, - imported(py, UNICODE_ENVIRONMENT_MODULE, "UnicodeEnvironment")?.call0()?, - ), + (inputs.getattr("effective_vars")?, unicode_environment(py)?), ( ( inputs.getattr("effective_target_name")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index 24d16e9b97..ca5b0aebec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -11,6 +11,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::compiled_project::loop_classes::model_loop_classes; +use crate::bindings::_helpers::model_config::contract_classes::contract_classes; +use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; use crate::bindings::_helpers::render::declaration_values::constant_declaration_sql; use crate::bindings::_helpers::render::macro_expansion_result::macro_expansion_classes; @@ -22,12 +24,9 @@ const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; -const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; -const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; +const COLLECTOR_MODULE: &str = "sqlbuild._native"; const PROJECT_VARS_MODULE: &str = "sqlbuild._native"; const MACRO_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; -const CONTRACT_CLASSES_MODULE: &str = - "sqlbuild.compiler.model_config.main._native_contract_classes"; const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; @@ -128,10 +127,7 @@ fn sql_hook_inputs<'py>( ), ("hook_files", discovered.getattr("sql_hook_files")?), ("effective_vars", context.getattr("effective_vars")?), - ( - "environment", - imported(py, UNICODE_ENVIRONMENT_MODULE, "UnicodeEnvironment")?.call0()?, - ), + ("environment", unicode_environment(py)?), ( "render_project_var", imported(py, PROJECT_VARS_MODULE, "render_project_var_text")?, @@ -276,11 +272,7 @@ pub(crate) fn build_model_inputs<'py>( let syntax: Bound<'py, PyAny> = context.getattr("sql_lexical_syntax")?; let native_loop: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "NativeModelLoop")?.call1(( - ( - validator, - imported(py, CONTRACT_CLASSES_MODULE, "native_contract_classes")?.call0()?, - classes, - ), + (validator, contract_classes(py)?.into_any(), classes), syntax.getattr("native_mapping")?, reserved_cursor_markers(), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs index 23fc340a8d..df0f86093d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -8,13 +8,13 @@ use crate::bindings::_helpers::compile_attachment::effective_config::project_var use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::frontier::cache_directory::cache_directory; use crate::bindings::_helpers::model_config::template_data::{early_model_templates, run_id}; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const FRONTIER_CONSTANTS_MODULE: &str = "sqlbuild.compiler.frontier.constants"; -const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; const DATABASE_KEY: &str = "database"; const SCHEMA_KEY: &str = "schema"; @@ -92,8 +92,7 @@ fn compile_cache_root<'py>( { return Ok(py.None().into_bound(py)); } - let root: Bound<'py, PyAny> = - imported(py, CACHE_DIRECTORY_MODULE, "compiler_cache_directory")?.call1((project_dir,))?; + let root: Bound<'py, PyAny> = cache_directory(project_dir)?; remove_retired_caches(&root)?; Ok(root) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs new file mode 100644 index 0000000000..b278291eed --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs @@ -0,0 +1,81 @@ +//! Compile reuse storage, fingerprint, environment and notice constants. + +/// The directory under the compiler cache directory holding the native reuse store. +pub(crate) const NATIVE_REUSE_DIRECTORY_NAME: &str = "project-reuse-v2"; +/// The reuse directory an older release kept stored compiles in. +pub(crate) const RETIRED_REUSE_DIRECTORY_NAME: &str = "project-reuse-v1"; +pub(crate) const REUSE_MAX_ENTRY_BYTES: i64 = 512 * 1024 * 1024; +pub(crate) const REUSE_MAX_STORED_ENTRIES: i64 = 8; +pub(crate) const REUSE_DISABLE_ENV_VAR: &str = "SQLBUILD_DISABLE_COMPILE_REUSE"; +pub(crate) const REUSE_DISABLE_VALUE: &str = "1"; +pub(crate) const REUSE_LOGGER_NAME: &str = "sqlbuild.compile.reuse"; +pub(crate) const TRACKED_ENVIRONMENT_PREFIXES: [&str; 2] = ["SQLBUILD_", "SQB_"]; +/// Sorted; tracked-prefix variables that never invalidate a stored compile. +pub(crate) const UNTRACKED_ENVIRONMENT_NAMES: [&str; 2] = [ + "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR", + "SQLBUILD_DISABLE_COMPILE_REUSE", +]; +pub(crate) const MISSING_ENVIRONMENT_VALUE: &str = "\0missing"; +pub(crate) const MISSING_FILE_DIGEST: &str = "\0missing"; +pub(crate) const PROJECT_CONFIG_FILENAMES: [&str; 2] = + ["sqlbuild_project.toml", "sqlbuild_project.yml"]; +/// The `sqlbuild._native` attribute the reuse check reads its racy window from. +pub(crate) const RACY_WINDOW_ATTRIBUTE: &str = "COMPILE_REUSE_RACY_WINDOW_NS"; +/// The `sqlbuild._native` attribute the reuse check reads its excluded root directories from. +pub(crate) const EXCLUDED_ROOT_ATTRIBUTE: &str = "COMPILE_REUSE_EXCLUDED_ROOT_DIRECTORIES"; +pub(crate) const RACY_WINDOW_NS: i64 = 2_000_000_000; +pub(crate) const MISSING_PATH_MTIME_NS: i64 = -1; +pub(crate) const PROJECT_ROOT_PATH_MTIME_NS: i64 = -2; +pub(crate) const TARGET_DIRECTORY_NAME: &str = "target"; +pub(crate) const COMPILED_DIRECTORY_NAME: &str = "compiled"; +pub(crate) const MILLISECONDS_PER_SECOND: f64 = 1000.0; +/// Sorted; top-level project directories the reuse snapshot skips. +pub(crate) const EXCLUDED_ROOT_DIRECTORIES: [&str; 16] = [ + ".cache", + ".fensu", + ".hg", + ".idea", + ".mypy_cache", + ".nox", + ".pytest_cache", + ".ruff_cache", + ".sqlbuild", + ".svn", + ".tox", + ".venv", + ".vscode", + "logs", + "target", + "venv", +]; +/// Sorted; directories the reuse snapshot skips at any depth. +pub(crate) const EXCLUDED_DIRECTORIES: [&str; 2] = [".git", "__pycache__"]; +pub(crate) const PRESENCE_ONLY_FILE_SUFFIXES: [&str; 2] = [".duckdb", ".duckdb.wal"]; +pub(crate) const REUSE_HIT_TIMING: &str = "project_reuse_hits"; +pub(crate) const REUSE_MISS_TIMING: &str = "project_reuse_misses"; +pub(crate) const REUSE_BYPASS_TIMING: &str = "project_reuse_bypasses"; +pub(crate) const REUSE_CHECK_TIMING: &str = "project_reuse_check_ms"; +pub(crate) const TOTAL_TIMING: &str = "total_ms"; +/// The reuse hit notice, split around its elapsed seconds. +pub(crate) const REUSE_HIT_MESSAGE: [&str; 2] = + ["Inputs unchanged; reused the previous compile (", " s)"]; +pub(crate) const REUSE_STORE_NOTICE_BYTES: u64 = 64 * 1024 * 1024; +pub(crate) const REUSE_STORE_NOTICE_PATHS: usize = 100_000; +pub(crate) const BYTES_PER_MEBIBYTE: f64 = 1024.0 * 1024.0; +pub(crate) const REUSE_STORE_DONE_PREFIX: &str = "Recorded compile for reuse ("; +pub(crate) const REUSE_STORE_SKIPPED_PREFIX: &str = + "Did not record compile for reuse: files changed while it ran ("; +pub(crate) const COMPUTE_LOG_DIRECTORY_NAME: &str = "logs"; +pub(crate) const COMPILE_COMMAND_NAME: &str = "compile"; +/// The required and optional keyword fields of `CompileReuseAttempt`. +pub(crate) const REUSE_ATTEMPT_FIELDS: (&[&str], &[&str]) = ( + &["outcome", "project_dir", "started"], + &[ + "check_ms", + "exit_code", + "entry_path", + "native", + "selected_target", + "json_output", + ], +); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs index a89f776184..73544bca28 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs @@ -1,4 +1,6 @@ pub(crate) mod artifact_batch; +pub(crate) mod constants; +pub(crate) mod models; pub(crate) mod prepared_artifacts; pub(crate) mod registration; pub(crate) mod report; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs new file mode 100644 index 0000000000..79d636aea0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs @@ -0,0 +1,51 @@ +//! The per-invocation compile reuse state carried from the reuse check into a full compile. + +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{IntoPyObjectExt, pyclass, pymethods}; + +use crate::bindings::_helpers::compile_command::constants::REUSE_ATTEMPT_FIELDS; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; + +/// Result of checking for a reusable compile, carried into a full compile on a miss. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct CompileReuseAttempt { + /// `hit`, `miss` or `bypass`. + outcome: Py, + project_dir: Py, + started: Py, + check_ms: Py, + exit_code: Py, + entry_path: Py, + /// The natively checked attempt, recorded natively after a miss. + native: Py, + selected_target: Py, + json_output: Py, +} + +#[pymethods] +impl CompileReuseAttempt { + #[new] + #[pyo3(signature = (**options))] + fn new(py: Python<'_>, options: Option>) -> PyResult { + let fields: KeywordArguments<'_> = + KeywordArguments::read(py, "CompileReuseAttempt", options, REUSE_ATTEMPT_FIELDS)?; + let field = |name: &str| -> PyResult> { Ok(fields.value(name)?.unbind()) }; + Ok(Self { + outcome: field("outcome")?, + project_dir: field("project_dir")?, + started: field("started")?, + check_ms: if fields.passed("check_ms")? { + field("check_ms")? + } else { + 0_i64.into_py_any(py)? + }, + exit_code: field("exit_code")?, + entry_path: field("entry_path")?, + native: field("native")?, + selected_target: field("selected_target")?, + json_output: fields.value_or_false("json_output")?.unbind(), + }) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs index dfc638391e..8e6f6d42e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs @@ -1,12 +1,15 @@ //! Register the `sqb compile` report, artifact writing and reuse recording. -use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_command::{ prepared_artifacts, report, reuse_attempt, reuse_recording, target_writer, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; report::register(module)?; reuse_attempt::register(module)?; reuse_recording::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index 30ba913b70..0305e5f21d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -6,60 +6,67 @@ use pyo3::exceptions::PyOSError; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyString}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyString}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_cli::replay::main::write_launch_record::write_launch_record as write_record_file; +use sqlbuild_cli::replay::constants::{LAUNCH_RECORD_DIRECTORY, LAUNCH_RECORD_FORMAT}; + use crate::bindings::_helpers::cli::launch_state::launch_record_path; +use crate::bindings::_helpers::compile_command::constants::{ + COMPILE_COMMAND_NAME, COMPILED_DIRECTORY_NAME, COMPUTE_LOG_DIRECTORY_NAME, + EXCLUDED_DIRECTORIES, EXCLUDED_ROOT_ATTRIBUTE, EXCLUDED_ROOT_DIRECTORIES, + MILLISECONDS_PER_SECOND, MISSING_ENVIRONMENT_VALUE, MISSING_FILE_DIGEST, MISSING_PATH_MTIME_NS, + NATIVE_REUSE_DIRECTORY_NAME, PRESENCE_ONLY_FILE_SUFFIXES, PROJECT_CONFIG_FILENAMES, + PROJECT_ROOT_PATH_MTIME_NS, RACY_WINDOW_ATTRIBUTE, RACY_WINDOW_NS, + RETIRED_REUSE_DIRECTORY_NAME, REUSE_BYPASS_TIMING, REUSE_CHECK_TIMING, REUSE_DISABLE_ENV_VAR, + REUSE_DISABLE_VALUE, REUSE_HIT_MESSAGE, REUSE_HIT_TIMING, REUSE_LOGGER_NAME, + REUSE_MAX_ENTRY_BYTES, REUSE_MAX_STORED_ENTRIES, REUSE_MISS_TIMING, TARGET_DIRECTORY_NAME, + TOTAL_TIMING, TRACKED_ENVIRONMENT_PREFIXES, UNTRACKED_ENVIRONMENT_NAMES, +}; +use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_driver::phases::{elapsed_ms, monotonic}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::compile_outputs::json_report::plain_json; +use crate::bindings::_helpers::frontier::cache_directory::cache_directory; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const REUSE_CONSTANTS: &str = "sqlbuild.cli.compile_reuse.constants"; -const REUSE_MODELS: &str = "sqlbuild.cli.compile_reuse.models"; -const REUSE_TYPES: &str = "sqlbuild.cli.compile_reuse.types"; +const REUSE_REQUEST: (&str, &str) = ("sqlbuild.cli.compile_reuse.models", "CompileReuseRequest"); const COMPILE_CONSTANTS: &str = "sqlbuild.compiler.compile.constants"; const RUNTIME_IDENTITY_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.runtime_identity"; const STARTUP_INPUTS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.startup_inputs"; -const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; const COMPUTE_LOG_TEE_MODULE: &str = "sqlbuild.runtime.compute_logs.classes.text_tee"; const OUTPUT_CAPTURE_CONSTANTS: &str = "sqlbuild.runtime.output_capture.constants"; -const SECONDS_FIELD: &str = "{seconds:.1f}"; -const HIT: &str = "HIT"; +const HIT: &str = "hit"; const CHECKED_HIT: &str = "hit"; const CHECKED_BYPASS: &str = "bypass"; -const MISS: &str = "MISS"; -const BYPASS: &str = "BYPASS"; - -fn constant<'py>(py: Python<'py>, name: &str) -> PyResult> { - imported(py, REUSE_CONSTANTS, name) -} - -fn sorted<'py>(value: Bound<'py, PyAny>) -> PyResult> { - imported(value.py(), "builtins", "sorted")?.call1((value,)) -} +const MISS: &str = "miss"; +const BYPASS: &str = "bypass"; fn listed<'py>(value: Bound<'py, PyAny>) -> PyResult> { Ok(PyList::new(value.py(), value.try_iter()?.collect::>>()?)?.into_any()) } fn reuse_logger(py: Python<'_>) -> PyResult> { - imported(py, "logging", "getLogger")?.call1((constant(py, "REUSE_LOGGER_NAME")?,)) + imported(py, "logging", "getLogger")?.call1((REUSE_LOGGER_NAME,)) } /// The reuse constants the native check, the stored compile and the launch record share. pub(crate) fn reuse_rules(py: Python<'_>, with_output_files: bool) -> PyResult> { let rules: Bound<'_, PyDict> = PyDict::new(py); - rules.set_item("excluded", sorted(constant(py, "EXCLUDED_DIRECTORIES")?)?)?; + rules.set_item("excluded", PyList::new(py, EXCLUDED_DIRECTORIES)?)?; rules.set_item( "excluded_root", - sorted(constant(py, "EXCLUDED_ROOT_DIRECTORIES")?)?, + imported(py, "builtins", "sorted")?.call1((imported( + py, + NATIVE_MODULE, + EXCLUDED_ROOT_ATTRIBUTE, + )?,))?, )?; rules.set_item( "presence_suffixes", - listed(constant(py, "PRESENCE_ONLY_FILE_SUFFIXES")?)?, + PyList::new(py, PRESENCE_ONLY_FILE_SUFFIXES)?, )?; let output_files: Bound<'_, PyAny> = imported(py, RUNTIME_IDENTITY_MODULE, "redirected_output_files")?.call0()?; @@ -68,11 +75,11 @@ pub(crate) fn reuse_rules(py: Python<'_>, with_output_files: bool) -> PyResult, with_output_files: bool) -> PyResult( .is_instance(&imported(py, COMPUTE_LOG_TEE_MODULE, "TextComputeLogTee")?)? { let logs: Bound<'py, PyAny> = resolved - .call_method1( - "__truediv__", - (constant(py, "COMPUTE_LOG_DIRECTORY_NAME")?,), - )? + .call_method1("__truediv__", (COMPUTE_LOG_DIRECTORY_NAME,))? .call_method0("resolve")?; let capture: Bound<'py, PyDict> = PyDict::new(py); capture.set_item("root", logs.str()?)?; - capture.set_item("command", constant(py, "COMPILE_COMMAND_NAME")?)?; + capture.set_item("command", COMPILE_COMMAND_NAME)?; capture.set_item("project_dir", resolved.str()?)?; capture.set_item("target", attempt.getattr("selected_target")?)?; capture.into_any() } else { py.None().into_bound(py) }; - let hit_message: Bound<'py, PyAny> = - constant(py, "REUSE_HIT_MESSAGE")?.call_method1("split", (SECONDS_FIELD,))?; let timings: Bound<'py, PyDict> = PyDict::new(py); for (key, name) in [ - ("hit", "REUSE_HIT_TIMING"), - ("miss", "REUSE_MISS_TIMING"), - ("bypass", "REUSE_BYPASS_TIMING"), - ("check", "REUSE_CHECK_TIMING"), - ("total", "TOTAL_TIMING"), + ("hit", REUSE_HIT_TIMING), + ("miss", REUSE_MISS_TIMING), + ("bypass", REUSE_BYPASS_TIMING), + ("check", REUSE_CHECK_TIMING), + ("total", TOTAL_TIMING), ] { - timings.set_item(key, constant(py, name)?)?; + timings.set_item(key, name)?; } let payload: Bound<'py, PyDict> = PyDict::new(py); - payload.set_item("format", constant(py, "LAUNCH_RECORD_FORMAT")?)?; + payload.set_item("format", LAUNCH_RECORD_FORMAT)?; payload.set_item("key", key)?; payload.set_item("project_dir", project_dir.str()?)?; payload.set_item("store_path", native.call_method0("store_path")?)?; @@ -180,7 +185,7 @@ fn launch_payload<'py>( payload.set_item("json_output", attempt.getattr("json_output")?)?; payload.set_item("rules", reuse_rules(py, false)?)?; payload.set_item("capture", capture)?; - payload.set_item("hit_message", listed(hit_message)?)?; + payload.set_item("hit_message", PyList::new(py, REUSE_HIT_MESSAGE)?)?; payload.set_item("timings", timings)?; Ok(payload) } @@ -213,8 +218,8 @@ pub(crate) fn write_launch_record(attempt: &Bound<'_, PyAny>) -> PyResult<()> { return Ok(()); } let mut directory: Bound<'_, PyAny> = attempt.getattr("project_dir")?; - for part in constant(py, "LAUNCH_RECORD_DIRECTORY")?.try_iter()? { - directory = directory.call_method1("__truediv__", (part?,))?; + for part in LAUNCH_RECORD_DIRECTORY { + directory = directory.call_method1("__truediv__", (part,))?; } if path .getattr("parent")? @@ -240,7 +245,7 @@ pub(crate) fn write_launch_record(attempt: &Bound<'_, PyAny>) -> PyResult<()> { } fn outcome<'py>(py: Python<'py>, name: &str) -> PyResult> { - imported(py, REUSE_TYPES, "CompileReuseOutcome")?.getattr(name) + name.into_bound_py_any(py) } /// A `CompileReuseAttempt` with `fields`. @@ -248,7 +253,7 @@ fn reuse_attempt<'py>( py: Python<'py>, fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { - call_with(&imported(py, REUSE_MODELS, "CompileReuseAttempt")?, fields) + call_with(py.get_type::().as_any(), fields) } /// The native reuse check request for this invocation. @@ -263,12 +268,8 @@ fn reuse_check<'py>( check.set_item("project_dir", project_dir)?; check.set_item( "store_directory", - imported(py, CACHE_DIRECTORY_MODULE, "compiler_cache_directory")? - .call1((imported(py, "pathlib", "Path")?.call1((project_dir,))?,))? - .call_method1( - "__truediv__", - (constant(py, "NATIVE_REUSE_DIRECTORY_NAME")?,), - )?, + cache_directory(&imported(py, "pathlib", "Path")?.call1((project_dir,))?)? + .call_method1("__truediv__", (NATIVE_REUSE_DIRECTORY_NAME,))?, )?; check.set_item("selected_target", request.getattr("selected_target")?)?; check.set_item( @@ -330,16 +331,13 @@ fn replayed<'py>( ) -> PyResult> { let py: Python<'py> = native.py(); let check_ms: i64 = elapsed_ms(py, started)?; - let mut timings: Vec<(Bound<'py, PyAny>, i64)> = Vec::new(); - for (name, counted) in [ - ("REUSE_HIT_TIMING", 1), - ("REUSE_MISS_TIMING", 0), - ("REUSE_BYPASS_TIMING", 0), - ("REUSE_CHECK_TIMING", check_ms), - ] { - timings.push((constant(py, name)?, counted)); - } - timings.push((constant(py, "TOTAL_TIMING")?, elapsed_ms(py, started)?)); + let timings: Vec<(&str, i64)> = vec![ + (REUSE_HIT_TIMING, 1), + (REUSE_MISS_TIMING, 0), + (REUSE_BYPASS_TIMING, 0), + (REUSE_CHECK_TIMING, check_ms), + (TOTAL_TIMING, elapsed_ms(py, started)?), + ]; let rows: Bound<'py, PyList> = if json_output { PyList::new(py, timings)? } else { @@ -357,16 +355,11 @@ fn replayed<'py>( Bound<'py, PyAny>, Bound<'py, PyAny>, ) = replay.extract()?; - let seconds: Bound<'py, PyAny> = check_ms - .into_bound_py_any(py)? - .call_method1("__truediv__", (constant(py, "MILLISECONDS_PER_SECOND")?,))?; + let [prefix, suffix] = REUSE_HIT_MESSAGE; + let seconds: f64 = check_ms as f64 / MILLISECONDS_PER_SECOND; print_stderr( py, - &constant(py, "REUSE_HIT_MESSAGE")?.call_method( - "format", - (), - Some(&keywords(py, &[("seconds", seconds)])?), - )?, + &format!("{prefix}{seconds:.1}{suffix}").into_bound_py_any(py)?, )?; for line in stderr_lines.try_iter()? { print_stderr(py, &line?)?; @@ -487,7 +480,7 @@ fn reuse_request<'py>(request: &Bound<'py, PyAny>) -> PyResult fields.push((name, request.getattr(name)?)); } fields.push(("profiling", profiling.into_bound_py_any(py)?)); - call_with(&imported(py, REUSE_MODELS, "CompileReuseRequest")?, &fields) + call_with(&imported(py, REUSE_REQUEST.0, REUSE_REQUEST.1)?, &fields) } /// Run one compile command request, replaying the previous compile when nothing changed. @@ -496,13 +489,19 @@ fn run_compile<'py>(request: &Bound<'py, PyAny>) -> PyResult> let py: Python<'py> = request.py(); let attempt: Bound<'py, PyAny> = attempt_compile_reuse(&reuse_request(request)?)?; let exit_code: Bound<'py, PyAny> = attempt.getattr("exit_code")?; - if attempt.getattr("outcome")?.is(&outcome(py, HIT)?) && !exit_code.is_none() { + if attempt.getattr("outcome")?.eq(outcome(py, HIT)?)? && !exit_code.is_none() { return Ok(exit_code); } imported(py, NATIVE_MODULE, "run_compile_command")?.call1((request, attempt)) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py: Python<'_> = module.py(); + module.add(RACY_WINDOW_ATTRIBUTE, RACY_WINDOW_NS)?; + module.add( + EXCLUDED_ROOT_ATTRIBUTE, + PyFrozenSet::new(py, EXCLUDED_ROOT_DIRECTORIES)?, + )?; module.add_function(wrap_pyfunction!(attempt_compile_reuse, module)?)?; module.add_function(wrap_pyfunction!(run_compile, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs index 9908bd9144..c0f5edb951 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs @@ -8,13 +8,17 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use pyo3::{IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use crate::bindings::_helpers::compile_command::constants::{ + BYTES_PER_MEBIBYTE, REUSE_BYPASS_TIMING, REUSE_CHECK_TIMING, REUSE_HIT_TIMING, + REUSE_LOGGER_NAME, REUSE_MISS_TIMING, REUSE_STORE_DONE_PREFIX, REUSE_STORE_NOTICE_BYTES, + REUSE_STORE_NOTICE_PATHS, REUSE_STORE_SKIPPED_PREFIX, +}; use crate::bindings::_helpers::compile_command::reuse_attempt::{reuse_rules, write_launch_record}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; use crate::bindings::_helpers::compile_outputs::reuse::artifact_digest; -const REUSE_CONSTANTS_MODULE: &str = "sqlbuild.cli.compile_reuse.constants"; const PROVIDER_SETTINGS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.provider_settings"; const MACRO_STORE_PATHS_MODULE: &str = "sqlbuild.compiler.macro_bridge.main._macro_store_module_paths"; @@ -260,28 +264,18 @@ pub(crate) fn compile_reuse_timings<'py>( let outcome: String = attempt.getattr("outcome")?.str()?.to_string(); let timings: Bound<'py, PyDict> = PyDict::new(py); for (name, counted) in [ - ("REUSE_HIT_TIMING", HIT_OUTCOME), - ("REUSE_MISS_TIMING", MISS_OUTCOME), - ("REUSE_BYPASS_TIMING", BYPASS_OUTCOME), + (REUSE_HIT_TIMING, HIT_OUTCOME), + (REUSE_MISS_TIMING, MISS_OUTCOME), + (REUSE_BYPASS_TIMING, BYPASS_OUTCOME), ] { - timings.set_item( - imported(py, REUSE_CONSTANTS_MODULE, name)?, - i64::from(outcome == counted), - )?; + timings.set_item(name, i64::from(outcome == counted))?; } - timings.set_item( - imported(py, REUSE_CONSTANTS_MODULE, "REUSE_CHECK_TIMING")?, - attempt.getattr("check_ms")?, - )?; + timings.set_item(REUSE_CHECK_TIMING, attempt.getattr("check_ms")?)?; Ok(timings) } fn reuse_logger(py: Python<'_>) -> PyResult> { - imported(py, "logging", "getLogger")?.call1((imported( - py, - REUSE_CONSTANTS_MODULE, - "REUSE_LOGGER_NAME", - )?,)) + imported(py, "logging", "getLogger")?.call1((REUSE_LOGGER_NAME,)) } /// Loaded module files, then the macro store's extra module paths, as Python lists them. @@ -315,46 +309,29 @@ fn module_paths(py: Python<'_>) -> PyResult> { /// The notice a long reuse record prints first, with when it started. fn start_notice(py: Python<'_>, native: &Bound<'_, PyAny>) -> PyResult> { - let pending: i64 = native.call_method0("pending_digest_bytes")?.extract()?; - let paths: i64 = native.call_method0("snapshot_paths")?.extract()?; - let constant = |name: &str| imported(py, REUSE_CONSTANTS_MODULE, name); - if pending < constant("REUSE_STORE_NOTICE_BYTES")?.extract::()? - && paths < constant("REUSE_STORE_NOTICE_PATHS")?.extract::()? - { + let pending: u64 = native.call_method0("pending_digest_bytes")?.extract()?; + let paths: usize = native.call_method0("snapshot_paths")?.extract()?; + if pending < REUSE_STORE_NOTICE_BYTES && paths < REUSE_STORE_NOTICE_PATHS { return Ok(None); } - let mebibytes: Bound<'_, PyAny> = pending - .into_bound_py_any(py)? - .call_method1("__truediv__", (constant("BYTES_PER_MEBIBYTE")?,))?; - let message: Bound<'_, PyAny> = constant("REUSE_STORE_START_MESSAGE")?.call_method( - "format", - (), - Some(&keywords( - py, - &[ - ("paths", paths.into_bound_py_any(py)?), - ("mebibytes", mebibytes), - ], - )?), - )?; - print_to(py, "stderr", &message)?; + let mebibytes: f64 = pending as f64 / BYTES_PER_MEBIBYTE; + let message: String = + format!("Recording compile for reuse ({paths} paths, {mebibytes:.0} MiB to hash)..."); + print_to(py, "stderr", &message.into_bound_py_any(py)?)?; Ok(Some(imported(py, "time", "monotonic")?.call0()?.extract()?)) } -fn finish_notice(py: Python<'_>, started: Option, message: &str) -> PyResult<()> { +fn finish_notice(py: Python<'_>, started: Option, prefix: &str) -> PyResult<()> { let Some(started) = started else { return Ok(()); }; let now: f64 = imported(py, "time", "monotonic")?.call0()?.extract()?; - let text: Bound<'_, PyAny> = imported(py, REUSE_CONSTANTS_MODULE, message)?.call_method( - "format", - (), - Some(&keywords( - py, - &[("seconds", (now - started).into_bound_py_any(py)?)], - )?), - )?; - print_to(py, "stderr", &text) + let seconds: f64 = now - started; + print_to( + py, + "stderr", + &format!("{prefix}{seconds:.1} s)").into_bound_py_any(py)?, + ) } /// What `write_reusable_compile` records about one full compile. @@ -478,9 +455,9 @@ fn record<'py>(facts: &CompileFacts<'py>) -> PyResult { py, notice_started, if outcome == SKIPPED { - "REUSE_STORE_SKIPPED_MESSAGE" + REUSE_STORE_SKIPPED_PREFIX } else { - "REUSE_STORE_DONE_MESSAGE" + REUSE_STORE_DONE_PREFIX }, )?; Ok(outcome == STORED) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 9e6bffcfb9..773c0ff562 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -48,9 +48,8 @@ const OUTPUT_MODELS_MODULE: &str = "sqlbuild.cli.output.models"; const CLI_EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; -const MODEL_CHAIN_MODULE: &str = "sqlbuild.compiler.planner.main.execution.sql_test_model_chain"; -const PLAN_ARTIFACTS_MODULE: &str = - "sqlbuild.compiler.planner.main.execution._plan_compiled_sql_test_artifacts"; +const MODEL_CHAIN_MODULE: &str = "sqlbuild._native"; +const PLAN_ARTIFACTS_MODULE: &str = "sqlbuild._native"; const COMPARISON_SQL_MODULE: &str = "sqlbuild.executor.testing.main.comparison_sql"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const TARGET_OPTIONS: (&[&str], &[&str]) = (&["target_dir", "adapter", "project"], &["manifest"]); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs new file mode 100644 index 0000000000..28137f8425 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs @@ -0,0 +1,238 @@ +//! The analysis cache context and the model analysis cache metrics. + +use pyo3::exceptions::{PyOSError, PyTypeError, PyValueError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PySlice, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_identity::json_values::{COMPACT, unicode_sorted_dumps}; + +const ANALYSIS_CACHE_VERSION: i64 = 13; +const ANALYSIS_ALGORITHM_FINGERPRINT: &str = "model-sql-analysis-v21-completed-star-shapes"; +const LOCAL_QUALNAME_MARKER: &str = ""; +const PACKAGE_NAME: &str = "sqlbuild"; +const UNKNOWN_VERSION: &str = "unknown"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const SQL_ANALYSIS_CONSTANTS_MODULE: &str = "sqlbuild.compiler.sql_analysis.constants"; +const METRIC_MODULE: &str = "sqlbuild.compiler.profiling.main._metric"; +const METADATA_MODULE: &str = "importlib.metadata"; +const CONTEXT_REQUIRED: [&str; 3] = ["root", "inference_profile", "allow_compact_analysis"]; +const CONTEXT_OPTIONAL: [&str; 2] = ["rich_type_inference", "signature_namespace"]; + +/// A reusable cache context, or `None` when the profile identity is unstable. +pub(crate) fn analysis_cache_context<'py>( + root: &Bound<'py, PyAny>, + inference_profile: &Bound<'py, PyAny>, + (allow_compact_analysis, rich_type_inference): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + signature_namespace: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = root.py(); + if root.is_none() { + return Ok(None); + } + let Some(profile) = inference_profile_payload(inference_profile)? else { + return Ok(None); + }; + let shared: Bound<'py, PyDict> = PyDict::new(py); + shared.set_item("algorithm", ANALYSIS_ALGORITHM_FINGERPRINT)?; + shared.set_item("cache_version", ANALYSIS_CACHE_VERSION)?; + shared.set_item("sqlbuild_version", package_version(py, "sqlbuild")?)?; + shared.set_item( + "polyglot_version", + package_version(py, "polyglot-sql-chio")?, + )?; + shared.set_item( + "python_version", + imported(py, "platform", "python_version_tuple")? + .call0()? + .get_item(PySlice::new(py, 0, 2, 1))?, + )?; + shared.set_item("allow_compact_analysis", allow_compact_analysis)?; + shared.set_item("rich_type_inference", rich_type_inference)?; + shared.set_item( + "type_checked_dialects", + imported(py, "builtins", "sorted")?.call1((imported( + py, + SQL_ANALYSIS_CONSTANTS_MODULE, + "TYPE_CHECKED_DIALECTS", + )?,))?, + )?; + shared.set_item("inference_profile", profile)?; + let digests = (|| -> PyResult<(String, String)> { + Ok(( + payload_digest(shared.as_any())?, + payload_digest(signature_namespace)?, + )) + })(); + let (shared_fingerprint, namespace) = match digests { + Ok(digests) => digests, + Err(error) if json_failure(py, &error) => return Ok(None), + Err(error) => return Err(error), + }; + let context = call_with( + &imported(py, COMPILE_MODELS_MODULE, "AnalysisCacheContext")?, + &[ + ("root", root.clone()), + ( + "shared_fingerprint", + PyString::new(py, &shared_fingerprint).into_any(), + ), + ( + "signature_namespace", + PyString::new(py, &namespace).into_any(), + ), + ], + ); + match context { + Ok(context) => Ok(Some(context)), + Err(error) if json_failure(py, &error) => Ok(None), + Err(error) => Err(error), + } +} + +/// Build a reusable cache context, or bypass when profile identity is unstable. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_analysis_cache_context<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult>> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_analysis_cache_context", + options, + (&CONTEXT_REQUIRED, &CONTEXT_OPTIONAL), + )?; + let rich: Bound<'py, PyAny> = if arguments.passed("rich_type_inference")? { + arguments.value("rich_type_inference")? + } else { + PyBool::new(py, true).to_owned().into_any() + }; + analysis_cache_context( + &arguments.value("root")?, + &arguments.value("inference_profile")?, + (&arguments.value("allow_compact_analysis")?, &rich), + &arguments.value("signature_namespace")?, + ) +} + +/// Record semantic analysis cache outcomes for one invocation. +pub(crate) fn record_analysis_cache_metrics( + py: Python<'_>, + (batch_hits, entry_hits, misses, bypasses): (usize, usize, usize, usize), +) -> PyResult<()> { + let record: Bound<'_, PyAny> = imported(py, METRIC_MODULE, "record_compile_metric")?; + for (metric, value) in [ + ("analysis_batch_cache_hits", batch_hits), + ("analysis_entry_cache_hits", entry_hits), + ("analysis_cache_misses", misses), + ("analysis_cache_bypasses", bypasses), + ] { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("metric", metric)?; + keywords.set_item("value", value)?; + record.call((), Some(&keywords))?; + } + Ok(()) +} + +/// Whether `error` is the `TypeError` or `ValueError` an unstable payload raises. +fn json_failure(py: Python<'_>, error: &PyErr) -> bool { + error.is_instance_of::(py) || error.is_instance_of::(py) +} + +/// The profile identity payload, or `None` when a nullability rule is not a stable module function. +fn inference_profile_payload<'py>( + profile: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = profile.py(); + let inspect = py.import("inspect")?; + let sorted: Bound<'py, PyAny> = imported(py, "builtins", "sorted")?; + let rules: Bound<'py, PyList> = PyList::empty(py); + let items: Bound<'py, PyAny> = sorted.call1((profile + .getattr("function_nullability_rules")? + .call_method0("items")?,))?; + for item in items.try_iter()? { + let (name, rule): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + if !inspect.call_method1("isfunction", (&rule,))?.is_truthy()? { + return Ok(None); + } + let module: Bound<'py, PyAny> = rule.getattr("__module__")?; + let qualname: Bound<'py, PyAny> = rule.getattr("__qualname__")?; + let stable: bool = module + .extract::()? + .strip_prefix(PACKAGE_NAME) + .is_some_and(|rest| rest.starts_with('.')) + && !qualname + .extract::()? + .contains(LOCAL_QUALNAME_MARKER) + && rule.getattr("__closure__")?.is_none() + && rule.getattr("__defaults__")?.is_none() + && rule.getattr("__kwdefaults__")?.is_none(); + if !stable { + return Ok(None); + } + let source: String = match inspect.call_method1("getsource", (&rule,)) { + Ok(source) => source.extract()?, + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + return Ok(None); + } + Err(error) => return Err(error), + }; + let entry: Bound<'py, PyDict> = PyDict::new(py); + entry.set_item("name", name)?; + entry.set_item("module", module)?; + entry.set_item("qualname", qualname)?; + entry.set_item("source_digest", sha256_hex(source.as_bytes()))?; + rules.append(entry)?; + } + let payload: Bound<'py, PyDict> = PyDict::new(py); + for field in [ + "sql_analysis_dialect", + "quoted_identifiers_ignore_case", + "semantic_known_functions", + "semantic_known_types", + ] { + payload.set_item(field, profile.getattr(field)?)?; + } + payload.set_item("function_nullability_rules", rules)?; + payload.set_item( + "function_return_types", + imported(py, "builtins", "dict")?.call1((sorted.call1((profile + .getattr("function_return_types")? + .call_method0("items")?,))?,))?, + )?; + Ok(Some(payload)) +} + +/// The SHA-256 of `json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True)`. +fn payload_digest(payload: &Bound<'_, PyAny>) -> PyResult { + let encoded: String = unicode_sorted_dumps(payload, COMPACT)?; + Ok(sha256_hex(encoded.as_bytes())) +} + +/// `importlib.metadata.version(package)`, or `unknown` when the package is not installed. +fn package_version(py: Python<'_>, package: &str) -> PyResult { + match imported(py, METADATA_MODULE, "version")?.call1((package,)) { + Ok(version) => version.extract(), + Err(error) + if error.is_instance(py, &imported(py, METADATA_MODULE, "PackageNotFoundError")?) => + { + Ok(UNKNOWN_VERSION.to_owned()) + } + Err(error) => Err(error), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_analysis_cache_context, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs index 635fbd2cd9..e21a80bc05 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -10,11 +10,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, within, }; -const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const COLLECTOR_MODULE: &str = "sqlbuild._native"; const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; const RECORD_ANALYSES_MODULE: &str = "sqlbuild._native"; -const EXPLICIT_REFERENCES_MODULE: &str = - "sqlbuild.compiler.compile._helpers.explicit_references.checks"; +const EXPLICIT_REFERENCES_MODULE: &str = "sqlbuild._native"; const ACTIVE_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; const RUN_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.classes.macro_bridge"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index d88b9f44d6..8114cefb8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -6,6 +6,7 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple} use sqlbuild_analysis::assembly::analysis_session::main::native_dialect::native_dialect; use crate::bindings::_helpers::analysis::binding_catalog::{BindingCatalog, ignores_quoted_case}; +use crate::bindings::_helpers::compile_driver::analysis_cache::analysis_cache_context; use crate::bindings::_helpers::compile_driver::assembly::AssemblyRequest; use crate::bindings::_helpers::compile_driver::python_calls::{ call_with, imported, keywords, within, @@ -13,7 +14,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; -const ANALYSIS_CACHE_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.cache"; pub(crate) const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._record"; const PROFILING_CPU_MODULE: &str = "sqlbuild.compiler.profiling.main._record_cpu"; @@ -411,14 +411,11 @@ fn analysis_cache<'py>( ("vars", inputs.getattr("effective_vars")?), ], )?; - call_with( - &imported(py, ANALYSIS_CACHE_MODULE, "build_analysis_cache_context")?, - &[ - ("root", request.analysis_cache_dir.bind(py).clone()), - ("inference_profile", context.profile.clone()), - ("allow_compact_analysis", allow_compact.clone()), - ("rich_type_inference", rich.clone()), - ("signature_namespace", namespace.into_any()), - ], - ) + Ok(analysis_cache_context( + request.analysis_cache_dir.bind(py), + &context.profile, + (allow_compact, rich), + namespace.as_any(), + )? + .unwrap_or_else(|| py.None().into_bound(py))) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs index 40601cab98..48343b56db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const COLLECTOR_MODULE: &str = "sqlbuild._native"; const DECIMAL_MODULE: &str = "decimal"; const DATACLASSES_MODULE: &str = "dataclasses"; const SQL_TEST_INPUT_FAILURE: &str = "input"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 30f86b80b8..c2cdee3133 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -5,6 +5,7 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyMapping, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_command::report::{ format_compile_json, format_compile_text, json_semantic_notice, }; @@ -22,10 +23,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::rules::results::format_skipped_type_proof_rules; -const REUSE_MODELS_MODULE: &str = "sqlbuild.cli.compile_reuse.models"; -const REUSE_TYPES_MODULE: &str = "sqlbuild.cli.compile_reuse.types"; +const BYPASS_OUTCOME: &str = "bypass"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._collect"; @@ -104,12 +103,9 @@ fn bypass_attempt<'py>( path_class.call_method0("cwd")? }; call_with( - &imported(py, REUSE_MODELS_MODULE, "CompileReuseAttempt")?, + py.get_type::().as_any(), &[ - ( - "outcome", - imported(py, REUSE_TYPES_MODULE, "CompileReuseOutcome")?.getattr("BYPASS")?, - ), + ("outcome", BYPASS_OUTCOME.into_pyobject(py)?.into_any()), ("project_dir", path_class.call1((project_dir,))?), ("started", started.into_pyobject(py)?.into_any()), ], @@ -154,8 +150,7 @@ fn recorded_compile<'py>( &imported(py, PROFILING_MODULE, "collect_compile_timings")?.call0()?, |detailed_timings| { within( - &imported(py, COMPILE_CONSTANTS_MODULE, "COMPILE_INPUT_READS")? - .call_method0("recording")?, + &imported(py, NATIVE_MODULE, "record_compile_input_reads")?.call0()?, |input_reads| { let artifact_writes: Py = new_artifact_recorder(py)?; let manager: Bound<'py, PyAny> = call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs new file mode 100644 index 0000000000..5b151c0955 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs @@ -0,0 +1,134 @@ +//! Collect compile diagnostics so one compile reports every violation, not only the first. + +use std::sync::Mutex; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{PyErr, pyclass, pyfunction, pymethods, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{context_variable, imported}; + +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const CONTEXT_VARIABLE_NAME: &str = "sqlbuild_collected_compile_diagnostics"; + +static ACTIVE_COLLECTOR: PyOnceLock> = PyOnceLock::new(); + +/// The context variable holding the collection diagnostics are reported into, if any. +fn active_collector(py: Python<'_>) -> PyResult> { + context_variable(py, &ACTIVE_COLLECTOR, CONTEXT_VARIABLE_NAME) +} + +/// Deduplicated compile diagnostics gathered while the collection is entered. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct CollectedCompileDiagnostics { + by_key: Py, + token: Mutex>>, +} + +#[pymethods] +impl CollectedCompileDiagnostics { + fn __enter__<'py>(slf: &Bound<'py, Self>) -> PyResult> { + let token: Bound<'_, PyAny> = + active_collector(slf.py())?.call_method1("set", (slf.clone(),))?; + *slf.get() + .token + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(token.unbind()); + Ok(slf.clone()) + } + + #[pyo3(signature = (*_exception))] + fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult { + let token: Option> = self + .token + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + if let Some(token) = token { + active_collector(py)?.call_method1("reset", (token,))?; + } + Ok(false) + } + + /// Record one diagnostic; a repeated key keeps the first diagnostic. + #[pyo3(signature = (*, key, diagnostic))] + fn add( + &self, + py: Python<'_>, + key: &Bound<'_, PyAny>, + diagnostic: &Bound<'_, PyAny>, + ) -> PyResult<()> { + self.by_key + .bind(py) + .call_method1("setdefault", (key, diagnostic))?; + Ok(()) + } + + /// The recorded diagnostics in deterministic key order. + #[getter] + fn diagnostics<'py>(&self, py: Python<'py>) -> PyResult> { + let by_key: &Bound<'py, PyDict> = self.by_key.bind(py); + let keys: Bound<'py, PyAny> = imported(py, "builtins", "sorted")?.call1((by_key,))?; + let diagnostics: Vec> = keys + .try_iter()? + .map(|key| by_key.as_any().get_item(key?)) + .collect::>()?; + PyTuple::new(py, diagnostics) + } +} + +/// A new collection; entering it collects diagnostics instead of raising the first one. +#[pyfunction] +pub(crate) fn collect_compile_diagnostics( + py: Python<'_>, +) -> PyResult> { + Bound::new( + py, + CollectedCompileDiagnostics { + by_key: PyDict::new(py).unbind(), + token: Mutex::new(None), + }, + ) +} + +/// Record a diagnostic in the active collection, or raise it when none is active. +#[pyfunction] +#[pyo3(signature = (*, key, diagnostic))] +pub(crate) fn report_compile_diagnostic( + key: &Bound<'_, PyAny>, + diagnostic: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = key.py(); + let collector: Bound<'_, PyAny> = active_collector(py)?.call_method0("get")?; + if let Ok(collector) = collector.cast::() { + return collector.get().add(py, key, diagnostic); + } + let location: Bound<'_, PyAny> = diagnostic.getattr("location")?; + let path: Bound<'_, PyAny> = diagnostic.getattr("path")?; + let pointer: String = if !location.is_none() { + format!( + "\n --> {}:{}", + location.getattr("path")?.call_method0("as_posix")?.str()?, + location.getattr("line")?.str()? + ) + } else if !path.is_none() { + format!("\n --> {}", path.call_method0("as_posix")?.str()?) + } else { + String::new() + }; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("code", diagnostic.getattr("code")?)?; + keywords.set_item("help", diagnostic.getattr("help")?)?; + let message: String = format!("{}{pointer}", diagnostic.getattr("message")?.str()?); + Err(PyErr::from_value( + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")? + .call((message,), Some(&keywords))?, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(collect_compile_diagnostics, module)?)?; + module.add_function(wrap_pyfunction!(report_compile_diagnostic, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs index 0bc33c1533..bccd5ac5c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs @@ -4,8 +4,118 @@ use std::collections::BTreeSet; use std::sync::{Mutex, MutexGuard, PoisonError}; use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; use pyo3::types::PyTuple; -use pyo3::{pyclass, pymethods}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; + +const RUN_ID_KEY: &str = "run.id"; + +/// The recorders of every compile currently recording, delivered to from every thread. +#[pyclass(frozen, module = "sqlbuild._native")] +struct CompileInputReadRegistry { + recorders: Mutex>>, +} + +impl CompileInputReadRegistry { + fn locked(&self) -> MutexGuard<'_, Vec>> { + self.recorders + .lock() + .unwrap_or_else(PoisonError::into_inner) + } + + fn recorders(&self, py: Python<'_>) -> Vec> { + self.locked() + .iter() + .map(|recorder| recorder.clone_ref(py)) + .collect() + } +} + +static REGISTRY: PyOnceLock> = PyOnceLock::new(); + +fn registry(py: Python<'_>) -> PyResult<&Py> { + REGISTRY.get_or_try_init(py, || { + Py::new( + py, + CompileInputReadRegistry { + recorders: Mutex::new(Vec::new()), + }, + ) + }) +} + +fn recorders(py: Python<'_>) -> PyResult>> { + Ok(registry(py)?.get().recorders(py)) +} + +/// Note that a template looked up one environment variable, present or not. +pub(crate) fn environment_read(py: Python<'_>, name: &str) -> PyResult<()> { + for recorder in recorders(py)? { + recorder.get().add_environment_name(name.to_owned()); + } + Ok(()) +} + +/// Note a CTX lookup; only the per-invocation run identity is volatile. +pub(crate) fn context_read(py: Python<'_>, name: &str) -> PyResult<()> { + if name != RUN_ID_KEY { + return Ok(()); + } + for recorder in recorders(py)? { + recorder.get().mark_run_id_read(); + } + Ok(()) +} + +/// Note a provider settings class instantiated from the environment and files. +#[pyfunction] +fn record_settings_class_read(settings_class: &Bound<'_, PyAny>) -> PyResult<()> { + for recorder in recorders(settings_class.py())? { + recorder.get().add_settings_class(settings_class); + } + Ok(()) +} + +/// Records environment variables and run-identity reads while entered. +#[pyclass(frozen, module = "sqlbuild._native")] +struct CompileInputRecording { + recorder: Mutex>>, +} + +#[pymethods] +impl CompileInputRecording { + fn __enter__<'py>(&self, py: Python<'py>) -> PyResult> { + let recorder: Bound<'py, CompileInputReads> = Bound::new(py, CompileInputReads::new())?; + registry(py)?.get().locked().push(recorder.clone().unbind()); + *self.recorder.lock().unwrap_or_else(PoisonError::into_inner) = + Some(recorder.clone().unbind()); + Ok(recorder) + } + + #[pyo3(signature = (*_exception))] + fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult { + if let Some(recorder) = self + .recorder + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take() + { + registry(py)? + .get() + .locked() + .retain(|item| !item.is(&recorder)); + } + Ok(false) + } +} + +/// Record template input reads from every thread until the returned block exits. +#[pyfunction] +fn record_compile_input_reads() -> CompileInputRecording { + CompileInputRecording { + recorder: Mutex::new(None), + } +} #[derive(Default)] struct Reads { @@ -83,5 +193,8 @@ impl CompileInputReads { } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(record_settings_class_read, module)?)?; + module.add_function(wrap_pyfunction!(record_compile_input_reads, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index d4ad6d5525..944e325e08 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -12,7 +12,7 @@ use crate::bindings::_helpers::model_config::template_data::run_id; const CALL_STORE_MODULE: &str = "sqlbuild.compiler.compile._helpers.macro_bridge.call_store"; const ATTACHMENT_MODULE: &str = "sqlbuild._native"; const DECLARATION_SCOPE_MODULE: &str = "sqlbuild._native"; -const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const COLLECTOR_MODULE: &str = "sqlbuild._native"; const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const ATTACH_CHECKS_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index 102f4a0e04..dabff08ac1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod analysis; +pub(crate) mod analysis_cache; pub(crate) mod assembly; pub(crate) mod assembly_analysis; pub(crate) mod assembly_checks; @@ -9,6 +10,7 @@ pub(crate) mod assembly_sql_tests; pub(crate) mod command; pub(crate) mod config_validation; pub(crate) mod deferral; +pub(crate) mod diagnostic_collector; pub(crate) mod diff; pub(crate) mod discovery; pub(crate) mod input_reads; @@ -16,6 +18,7 @@ pub(crate) mod inputs; pub(crate) mod phases; pub(crate) mod pipeline; pub(crate) mod pipeline_entries; +pub(crate) mod plan_result; pub(crate) mod project; pub(crate) mod python_calls; pub(crate) mod python_plan_entries; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index 5f8f1e40c2..abd42d48c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -20,7 +20,7 @@ const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const CLI_CONSTANTS_MODULE: &str = "sqlbuild.cli.commands.constants"; +const EMPTY_DAG_PATH: &str = ""; const TARGET_DIRECTORY: &str = "target"; const DEFAULT_DAG_FILE: &str = "sqlbuild_dag.json"; @@ -90,8 +90,7 @@ pub(crate) fn resolved_dag_path<'py>( dag_path: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = project_dir.py(); - let empty: Bound<'py, PyAny> = imported(py, CLI_CONSTANTS_MODULE, "EMPTY_DAG_PATH")?; - if dag_path.eq(&empty)? { + if dag_path.eq(EMPTY_DAG_PATH)? { return target_dir(project_dir)?.call_method1("__truediv__", (DEFAULT_DAG_FILE,)); } let path: Bound<'py, PyAny> = imported(py, "pathlib", "Path")?.call1((dag_path,))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index 2a71c1c096..8bf92a1d30 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -8,6 +8,9 @@ use crate::bindings::_helpers::compile_driver::phases::monotonic; use crate::bindings::_helpers::compile_driver::pipeline_entries::{ compile_analysis_selection, project_graph, }; +use crate::bindings::_helpers::compile_driver::plan_result::{ + PlanningInputs, connection_with_hooks, pipeline_result, +}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; @@ -21,13 +24,11 @@ const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; -const TIMING_TRACKER_MODULE: &str = "sqlbuild.diagnostics.classes.build_phase_timing_tracker"; +const TIMING_TRACKER_MODULE: &str = "sqlbuild._native"; const RULE_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; const RULE_THREADS_MODULE: &str = "sqlbuild.rule_engine._helpers.run.rule_threads"; const RULE_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; -const OPEN_CONNECTION_MODULE: &str = "sqlbuild.runtime.contracts.main.open_connection"; const RUNTIME_MODELS_MODULE: &str = "sqlbuild.runtime.contracts.models"; -const PLAN_MODULE: &str = "sqlbuild.compiler.pipeline.main._plan_result"; const GENERIC_DIALECT: &str = "generic"; /// The current build's phase timing tracker, when one is recording. @@ -339,25 +340,18 @@ fn planned<'py>( let started: f64 = monotonic(py)?; let mut connection: Option> = None; let result: PyResult> = (|| { - let opened: Bound<'py, PyAny> = call_with( - &imported(py, OPEN_CONNECTION_MODULE, "open_connection_with_hooks")?, - &[ - ("adapter", adapter.clone()), - ("connection_config", compiled.getattr("connection_config")?), - ("hooks", hooks.clone()), - ], - )?; + let opened: Bound<'py, PyAny> = + connection_with_hooks(adapter, &compiled.getattr("connection_config")?, hooks)?; connection = Some(opened.clone()); - call_with( - &imported(py, PLAN_MODULE, "build_pipeline_result")?, - &[ - ("discovered_inputs", discovered.clone()), - ("adapter", adapter.clone()), - ("connection", opened), - ("project", compiled.getattr("project")?), - ("options", options.clone()), - ("on_progress", hooks.getattr("on_progress")?), - ], + pipeline_result( + &PlanningInputs { + discovered_inputs: discovered, + adapter, + connection: &opened, + }, + &compiled.getattr("project")?, + options, + &hooks.getattr("on_progress")?, ) })(); if let Some(connection) = connection.filter(|connection| !connection.is_none()) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs new file mode 100644 index 0000000000..2b188c660f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -0,0 +1,293 @@ +//! Plan a compiled project over an open warehouse connection, and open that connection. + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::phases::monotonic; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const EXECUTION_PLAN_MODULE: &str = "sqlbuild._native"; +const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const MATERIALIZATIONS_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.materializations"; +const RUN_SELECTION_MODULE: &str = "sqlbuild.compiler.python_nodes.main._run_selection"; +const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; +const SINGLE_CONNECTION: i64 = 1; + +/// The deferred relation locations and the warehouse relations behind them, when deferring. +type Deferral<'py> = (Bound<'py, PyAny>, Bound<'py, PyAny>); + +/// The planning inputs: the discovered project, the adapter and its open connection. +pub(crate) struct PlanningInputs<'a, 'py> { + pub(crate) discovered_inputs: &'a Bound<'py, PyAny>, + pub(crate) adapter: &'a Bound<'py, PyAny>, + pub(crate) connection: &'a Bound<'py, PyAny>, +} + +/// Plan the compiled project over an open connection and attach Python node plans. +pub(crate) fn pipeline_result<'py>( + inputs: &PlanningInputs<'_, 'py>, + project: &Bound<'py, PyAny>, + options: &Bound<'py, PyAny>, + on_progress: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let native = py.import(NATIVE_MODULE)?; + let discovered: &Bound<'py, PyAny> = inputs.discovered_inputs; + let select: Bound<'py, PyAny> = options.getattr("select")?; + let exclude: Bound<'py, PyAny> = options.getattr("exclude")?; + let (deferred_locations, deferred_relations): Deferral<'py> = + deferral(inputs, project, &options.getattr("defer_to")?)?; + let empty = || -> PyResult> { Ok(PyFrozenSet::empty(py)?.into_any()) }; + let mut selected_sql_keys: Bound<'py, PyAny> = py.None().into_bound(py); + let mut selected_python_node_names: Bound<'py, PyAny> = empty()?; + let mut python_read_source_names: Bound<'py, PyAny> = empty()?; + let mut run_selection: Bound<'py, PyAny> = py.None().into_bound(py); + let sql_test_selection: Bound<'py, PyAny> = options.getattr("sql_test_selection")?; + if !sql_test_selection.getattr("models_selected")?.is_truthy()? { + selected_sql_keys = empty()?; + } else if options + .getattr("resolve_python_run_selectors")? + .is_truthy()? + { + let graph: Bound<'py, PyAny> = call_with( + &native.getattr("build_project_graph_from_compiled_project")?, + &[("project", project.clone())], + )?; + run_selection = call_with( + &imported( + py, + RUN_SELECTION_MODULE, + "resolve_python_sql_run_selection_from_inputs", + )?, + &[ + ("select", select.clone()), + ("exclude", exclude.clone()), + ("project_graph", graph), + ("discovered_inputs", discovered.clone()), + ], + )?; + selected_sql_keys = run_selection.getattr("sql_keys")?; + selected_python_node_names = run_selection.getattr("python_node_names")?; + python_read_source_names = call_with( + &native.getattr("python_read_source_names_for")?, + &[ + ("discovered_inputs", discovered.clone()), + ("python_node_names", selected_python_node_names.clone()), + ], + )?; + } + let planner_models = py.import(PLANNER_MODELS_MODULE)?; + let selection: Bound<'py, PyAny> = call_with( + &planner_models.getattr("PlannerSelection")?, + &[ + ("select", select), + ("exclude", exclude), + ("selected_keys", selected_sql_keys), + ("python_read_source_names", python_read_source_names), + ("sql_test_selection", sql_test_selection), + ], + )?; + let overrides: Bound<'py, PyAny> = call_with( + &planner_models.getattr("PlannerOverrides")?, + &option_fields( + options, + &[ + "cursor_overrides", + "full_refresh", + "reload_sources", + "max_microbatches", + ], + )?, + )?; + let mut deferral_fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("deferred_locations", deferred_locations), + ("deferred_relations", deferred_relations), + ]; + deferral_fields.extend(option_fields( + options, + &["defer_sources_to", "source_deferral_enabled"], + )?); + let deferral: Bound<'py, PyAny> = + call_with(&planner_models.getattr("DeferralInputs")?, &deferral_fields)?; + let policies: Bound<'py, PyAny> = call_with( + &planner_models.getattr("PlannerPolicies")?, + &option_fields( + options, + &[ + "auto_load_sources", + "selection_diagnostics", + "plan_sql_tests", + "record_migration_fingerprints", + ], + )?, + )?; + let plan_output: Bound<'py, PyAny> = call_with( + &imported(py, EXECUTION_PLAN_MODULE, "build_execution_plan")?, + &[ + ("project", project.clone()), + ("adapter", inputs.adapter.clone()), + ("connection", inputs.connection.clone()), + ("selection", selection), + ("overrides", overrides), + ("deferral", deferral), + ("policies", policies), + ("on_progress", on_progress.clone()), + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + let custom_materializations: Bound<'py, PyAny> = + imported(py, MATERIALIZATIONS_MODULE, "load_custom_materializations")? + .call1((discovered.getattr("materialization_files")?,))?; + let python_outputs: Bound<'py, PyAny> = call_with( + &native.getattr("build_python_run_plan_outputs")?, + &[ + ("discovered_inputs", discovered.clone()), + ("plan_output", plan_output), + ("run_selection", run_selection), + ("selected_python_node_names", selected_python_node_names), + ], + )?; + call_with( + &imported(py, PIPELINE_MODELS_MODULE, "CompilePipelineResult")?, + &[ + ("project", project.clone()), + ("plan_output", python_outputs.getattr("plan_output")?), + ("custom_materializations", custom_materializations), + ( + "python_node_names", + python_outputs.getattr("selected_python_node_names")?, + ), + ( + "python_plan_entries", + python_outputs.getattr("python_plan_entries")?, + ), + ], + ) +} + +/// `(name, options.name)` for each of `names`. +fn option_fields<'a, 'py>( + options: &Bound<'py, PyAny>, + names: &[&'a str], +) -> PyResult)>> { + names + .iter() + .map(|name| Ok((*name, options.getattr(*name)?))) + .collect() +} + +/// The deferred locations and relations of `defer_to`, or two `None`s when not deferring. +fn deferral<'py>( + inputs: &PlanningInputs<'_, 'py>, + project: &Bound<'py, PyAny>, + defer_to: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + if defer_to.is_none() { + return Ok((py.None().into_bound(py), py.None().into_bound(py))); + } + let native = py.import(NATIVE_MODULE)?; + let adapter: &Bound<'py, PyAny> = inputs.adapter; + let target_config: Bound<'py, PyAny> = call_with( + &native.getattr("resolve_deferred_target_config")?, + &[ + ("discovered_inputs", inputs.discovered_inputs.clone()), + ("defer_to", defer_to.clone()), + ( + "current_target_name", + project.getattr("effective_target_name")?, + ), + ], + )?; + let locations: Bound<'py, PyAny> = call_with( + &native.getattr("build_deferred_locations")?, + &[ + ("project", project.clone()), + ("deferred_target_config", target_config), + ("effective_vars", project.getattr("effective_vars")?), + ("default_schema", adapter.call_method0("default_schema")?), + ( + "default_database", + adapter.call_method0("default_database")?, + ), + ( + "render_qualified_name", + adapter.getattr("render_qualified_name")?, + ), + ], + )?; + let relations: Bound<'py, PyAny> = call_with( + &native.getattr("gather_deferred_relations")?, + &[ + ("adapter", adapter.clone()), + ("connection", inputs.connection.clone()), + ("deferred_locations", locations.clone()), + ], + )?; + Ok((locations, relations)) +} + +/// Open one adapter connection and report its lifecycle through `hooks`. +pub(crate) fn connection_with_hooks<'py>( + adapter: &Bound<'py, PyAny>, + connection_config: &Bound<'py, PyAny>, + hooks: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let on_start: Bound<'py, PyAny> = hooks.getattr("on_connection_start")?; + if !on_start.is_none() { + on_start.call1((SINGLE_CONNECTION,))?; + } + let started: f64 = monotonic(py)?; + let elapsed = |started: f64| -> PyResult> { + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("elapsed_seconds", monotonic(py)? - started)?; + Ok(keywords) + }; + let connection: Bound<'py, PyAny> = match adapter.call_method1("connect", (connection_config,)) + { + Ok(connection) => connection, + Err(error) => { + let on_error: Bound<'py, PyAny> = hooks.getattr("on_connection_error")?; + if error.is_instance_of::(hooks.py()) && !on_error.is_none() { + on_error.call((SINGLE_CONNECTION,), Some(&elapsed(started)?))?; + } + return Err(error); + } + }; + let on_complete: Bound<'py, PyAny> = hooks.getattr("on_connection_complete")?; + if !on_complete.is_none() { + on_complete.call((SINGLE_CONNECTION,), Some(&elapsed(started)?))?; + } + Ok(connection) +} + +/// Open one adapter connection and report its lifecycle through hooks. +#[pyfunction] +#[pyo3(signature = (**options))] +fn open_connection_with_hooks<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "open_connection_with_hooks", + options, + (&["adapter", "connection_config", "hooks"], &[]), + )?; + connection_with_hooks( + &arguments.value("adapter")?, + &arguments.value("connection_config")?, + &arguments.value("hooks")?, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(open_connection_with_hooks, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs index 22c3e7f7b1..7d9561859a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs @@ -1,7 +1,8 @@ //! Calling the Python stages the native driver still runs: imports, keyword calls and `with`. use pyo3::exceptions::PyTypeError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString}; /// `module.name`, imported. @@ -109,6 +110,11 @@ impl<'py> KeywordArguments<'py> { Ok(Self { values }) } + /// Whether the argument `name` was passed. + pub(crate) fn contains(&self, name: &str) -> PyResult { + self.values.contains(name) + } + /// The argument `name`, or `None` when it was not passed. pub(crate) fn value(&self, name: &str) -> PyResult> { let py: Python<'py> = self.values.py(); @@ -137,3 +143,19 @@ impl<'py> KeywordArguments<'py> { self.values.contains(name) } } + +/// The `contextvars.ContextVar(name, default=None)` kept in `cell`, created once per interpreter. +pub(crate) fn context_variable<'py>( + py: Python<'py>, + cell: &'static PyOnceLock>, + name: &str, +) -> PyResult> { + cell.get_or_try_init(py, || { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("default", py.None())?; + imported(py, "contextvars", "ContextVar")? + .call((name,), Some(&keywords)) + .map(Bound::unbind) + }) + .map(|variable| variable.bind(py).clone()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs index 050f838115..917edd104b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs @@ -3,12 +3,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::compile_driver::{ - analysis, assembly, command, deferral, diff, discovery, input_reads, inputs, pipeline, - pipeline_entries, project, python_plan_entries, relation_targets, selection_phases, targets, + analysis, analysis_cache, assembly, command, deferral, diagnostic_collector, diff, discovery, + input_reads, inputs, pipeline, pipeline_entries, plan_result, project, python_plan_entries, + relation_targets, selection_phases, targets, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { analysis::register(module)?; + analysis_cache::register(module)?; assembly::register(module)?; command::register(module)?; project::register(module)?; @@ -17,7 +19,9 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { discovery::register(module)?; pipeline::register(module)?; pipeline_entries::register(module)?; + plan_result::register(module)?; deferral::register(module)?; + diagnostic_collector::register(module)?; diff::register(module)?; python_plan_entries::register(module)?; relation_targets::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index 5234f47d92..11ae284645 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -11,10 +11,9 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const SPLIT_SELECTORS_MODULE: &str = "sqlbuild._native"; -const PLANNER_SCOPES_MODULE: &str = - "sqlbuild.compiler.planner.main.selection._resolve_planner_scopes"; -const STATIC_SCOPE_MODULE: &str = "sqlbuild.compiler.planner.main.commands._scope"; -const STATIC_RELATIONS_MODULE: &str = "sqlbuild.compiler.planner.main.commands._relations"; +const PLANNER_SCOPES_MODULE: &str = "sqlbuild._native"; +const STATIC_SCOPE_MODULE: &str = "sqlbuild._native"; +const STATIC_RELATIONS_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; /// Whether a selection names nothing to select or exclude. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index ab172b2417..d64171676c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -30,7 +30,7 @@ const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild._native"; -const SETTING_HELP_MODULE: &str = "sqlbuild.errors.setting_help.main.setting_help"; +const SETTING_HELP_MODULE: &str = "sqlbuild._native"; /// Connection settings that identify the warehouse two targets write to. const CONNECTION_IDENTITY_KEYS: [&str; 7] = [ "account", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs index ba5e0ba839..d60a1188ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs @@ -12,7 +12,6 @@ const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; -const CONFIG_ERROR_MODULE: &str = "sqlbuild.compiler.model_config.main._native_config_error"; const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; @@ -443,10 +442,6 @@ pub(crate) fn model_loop_classes<'py>( "sql_hook_entry", class(py, DISCOVERY_MODELS_MODULE, "SqlHookEntry")?, ), - ( - "config_error", - imported(py, CONFIG_ERROR_MODULE, "native_config_error")?, - ), ( "audit_factories", by_attribute(audit_factories, "name")?.into_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs index a208ee6dd8..a9a455c55b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs @@ -55,6 +55,7 @@ use crate::bindings::_helpers::compiled_project::sql_test_loop::{ AttachedSqlTests, SqlTestSettings, TestFileRow, TestScope, }; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; +use crate::bindings::_helpers::model_config::config_errors::raised_config_exception; use crate::bindings::_helpers::model_config::header_metadata::ContractClasses; use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; use crate::bindings::_helpers::render::declaration_expansion::{ @@ -100,8 +101,6 @@ const VALIDATE_METHOD: &str = "validate"; const FILE_PATH_ARGUMENT: &str = "file_path"; const CONSTANT_TEXT_CALLBACK: &str = "constant_text"; const HELP_ARGUMENT: &str = "help"; -const ERROR_ARGUMENT: &str = "error"; -const VALUES_ARGUMENT: &str = "values"; const CODE_ARGUMENT: &str = "code"; const ANALYSIS_CONFLICT: &str = "MODEL sql_analysis conflicts with legacy sql_validation"; const ANALYSIS_CONFLICT_CODE: &str = "P003"; @@ -122,8 +121,6 @@ pub(crate) struct LoopClasses<'py> { pub(crate) usage_record: Bound<'py, PyAny>, pub(crate) compile_error: Bound<'py, PyAny>, pub(crate) sql_hook_entry: Bound<'py, PyAny>, - /// `native_config_error`: the exception a native config error raises as. - pub(crate) config_error: Bound<'py, PyAny>, /// Discovered audit factories by name. pub(crate) audit_factories: Bound<'py, PyDict>, /// `UsageKind.GENERATED`, for usages an enum-typed column generates. @@ -816,13 +813,12 @@ impl NativeModelLoop { (rows, declared_columns, unmanaged, declared), ), )?; - validation_outcome(context, &outcome, external, values) + validation_outcome(&outcome, external, values) } } /// `_raise_validation_outcome`: nothing, the validator's error, or a rejected reference's error. pub(crate) fn validation_outcome( - context: &LoopContext<'_, '_>, outcome: &Bound<'_, PyAny>, external: Vec>, values: &Bound<'_, PyDict>, @@ -835,7 +831,7 @@ pub(crate) fn validation_outcome( { return Err(error); } - Err(config_error(context.classes, outcome, Some(values))) + Err(config_error(outcome, Some(values))) } /// `applied_interpolation`: record the interpolation's template reads, then its SQL or error. @@ -900,21 +896,8 @@ pub(crate) fn helped_compile_error( } /// The Python exception a native config error raises as, given the values it names. -pub(crate) fn config_error( - classes: &LoopClasses<'_>, - error: &Bound<'_, PyAny>, - values: Option<&Bound<'_, PyDict>>, -) -> PyErr { - let raised = (|| { - let keywords: Bound<'_, PyDict> = PyDict::new(error.py()); - keywords.set_item(ERROR_ARGUMENT, error)?; - keywords.set_item(VALUES_ARGUMENT, values)?; - classes.config_error.call((), Some(&keywords)) - })(); - match raised { - Ok(instance) => PyErr::from_value(instance), - Err(failure) => failure, - } +pub(crate) fn config_error(error: &Bound<'_, PyAny>, values: Option<&Bound<'_, PyDict>>) -> PyErr { + raised_config_exception(error, values.map(Bound::as_any)) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs index 5e754d3fdd..0a400c0a40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs @@ -76,7 +76,7 @@ impl<'py> ModelConfigs<'py> { let builder: &NativeModelConfigBuilder = classes.builder.get(); let matched: Option = builder .selected_path_default(py, relative_path)? - .map_err(|error| config_error(context.classes, error.bind(py).as_any(), None))?; + .map_err(|error| config_error(error.bind(py).as_any(), None))?; let metadata: Option> = reusable_metadata(header)?; let reusable: bool = match metadata { Some(_) => self.reusable(context, &matched)?, @@ -94,7 +94,7 @@ impl<'py> ModelConfigs<'py> { let built: BuiltConfig<'py> = match built { ConfigBuild::Built(built) => built, ConfigBuild::Rejected(error) => { - return Err(config_error(context.classes, error.bind(py).as_any(), None)); + return Err(config_error(error.bind(py).as_any(), None)); } }; let fields: Bound<'py, PyDict> = config_fields(classes, &built, matched.as_deref())?; @@ -113,7 +113,7 @@ impl<'py> ModelConfigs<'py> { let builder: &NativeModelConfigBuilder = context.classes.config.builder.get(); let values: Bound<'py, PyDict> = builder .inherited_values(py, matched.as_deref())? - .map_err(|error| config_error(context.classes, error.bind(py).as_any(), None))?; + .map_err(|error| config_error(error.bind(py).as_any(), None))?; let checked: Bound<'py, PyTuple> = PyTuple::new( py, [values.as_any(), builder.target_namespace(py)?.as_any()], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs index dc7a179449..f73e9a239a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs @@ -99,7 +99,7 @@ pub(crate) fn header_schema<'py>( let metadata = parse_header_metadata(&node(columns), &node(audits), source.relative_path); let parsed_columns = metadata .columns - .map_err(|error| raised_config_error(py, context, error))?; + .map_err(|error| raised_config_error(py, error))?; let local_columns: Bound<'py, PyTuple> = column_tuple( py, context.contracts, @@ -110,13 +110,13 @@ pub(crate) fn header_schema<'py>( Some(schema) => merged_columns(context, schema, &local_columns, model)?, None => local_columns, }; - let families = dynamic_families(py, context, &node(dynamic), model)?; + let families = dynamic_families(py, &node(dynamic), model)?; let parsed_audits = metadata .audits - .map_err(|error| raised_config_error(py, context, error))?; + .map_err(|error| raised_config_error(py, error))?; let available: Vec = context.classes.audit_factories.keys().extract()?; let references: Vec = audit_factory_references(&node(factories), model, &available) - .map_err(|error| raised_config_error(py, context, error))?; + .map_err(|error| raised_config_error(py, error))?; let direct_audits: Bound<'py, PyTuple> = audit_tuple(py, context.contracts, &parsed_audits)?; let (generated, generated_identities) = factory_audits(py, context, &references)?; let direct_identities: Vec = direct_audits @@ -124,14 +124,14 @@ pub(crate) fn header_schema<'py>( .map(|audit| audit_identity(&audit)) .collect::>()?; merged_audit_identities(&direct_identities, &generated_identities, model) - .map_err(|error| raised_config_error(py, context, error))?; + .map_err(|error| raised_config_error(py, error))?; let migrations: Vec = columns .iter() .map(|column| column_migration(&column)) .collect::>()?; if let Some(error) = column_migration_error(source.name, source.values, &migrations)? { let error = native_config_error(py, error)?.into_bound(py).into_any(); - return Err(config_error(context.classes, &error, Some(source.values))); + return Err(config_error(&error, Some(source.values))); } let mut type_enforcement: bool = !families.is_empty(); for column in columns.iter() { @@ -180,11 +180,10 @@ pub(crate) fn header_schema<'py>( /// The header's dynamic column families, each `name_pattern` compiled natively. fn dynamic_families( py: Python<'_>, - context: &LoopContext<'_, '_>, raw: &PyNode<'_>, model: (&str, &str), ) -> PyResult> { - parse_dynamic_columns(raw, model).map_err(|error| raised_config_error(py, context, error)) + parse_dynamic_columns(raw, model).map_err(|error| raised_config_error(py, error)) } /// The header description: absent, or a non-blank string by Python's `str.strip`. @@ -209,11 +208,10 @@ fn description( fn raised_config_error( py: Python<'_>, - context: &LoopContext<'_, '_>, error: sqlbuild_model_config::errors::ConfigError, ) -> pyo3::PyErr { match native_config_error(py, error) { - Ok(native) => config_error(context.classes, native.bind(py).as_any(), None), + Ok(native) => config_error(native.bind(py).as_any(), None), Err(failure) => failure, } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs index 8dc7b26e73..a0a5504022 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs @@ -152,7 +152,7 @@ pub(crate) fn expanded_hooks<'py>( .validator .bind(py) .call_method1(REFERENCES_METHOD, ((model.name, model.relative_path), rows))?; - validation_outcome(context, &outcome, external, &PyDict::new(py))?; + validation_outcome(&outcome, external, &PyDict::new(py))?; reject_hook_intrinsics(model_loop, context, model.name, &expanded)?; Ok(ExpandedHooks { values: expanded, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs index 286209cd78..01ab413136 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs @@ -16,6 +16,7 @@ use crate::bindings::_helpers::compiled_project::model_loop_macros::{ }; use crate::bindings::_helpers::compiled_project::source_views::{AttachedSource, replaced}; use crate::bindings::_helpers::model_config::config_errors::native_config_error; +use crate::bindings::_helpers::model_config::config_errors::raised_config_exception; use crate::bindings::_helpers::model_config::config_templates::{ PythonHost, Stop, TemplateSources, expanded, template_error, }; @@ -350,23 +351,11 @@ pub(crate) fn expanded_template<'py>( let error: Bound<'py, PyAny> = native_config_error(py, template_error(&error, label))? .into_bound(py) .into_any(); - Err(raised_config_error(context, &error)) + Err(raised_config_exception(&error, None)) } } } -fn raised_config_error(context: &LoopContext<'_, '_>, error: &Bound<'_, PyAny>) -> PyErr { - let raised = (|| { - let keywords: Bound<'_, PyDict> = PyDict::new(error.py()); - keywords.set_item("error", error)?; - context.classes.config_error.call((), Some(&keywords)) - })(); - match raised { - Ok(instance) => PyErr::from_value(instance), - Err(failure) => failure, - } -} - /// `expand_source_column_templates`. fn expanded_column( context: &LoopContext<'_, '_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index 78a94477a0..0659bfa7f6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -21,6 +21,7 @@ use sqlbuild_core::contract_command::models::{ use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::contract_command::write::{ContractWrite, write_contracts}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -29,7 +30,6 @@ const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const ADOPTION_MODELS: &str = "sqlbuild.compiler.contract_adoption.models"; -const WRITE_MODULE: &str = "sqlbuild.compiler.contract_adoption.main._write"; const GENERATE_ACTION: &str = "generate"; /// A `CliUserError` with its code. @@ -258,18 +258,14 @@ fn write_updates<'py>( let status: Bound<'py, PyAny> = progress.status()?; let started: Instant = Instant::now(); status.call_method1("start", ("Updating repository contracts...",))?; - let updated: Bound<'py, PyAny> = match call_with( - &imported(py, WRITE_MODULE, "write_contracts")?, - &[ - ("project_dir", project_dir.clone()), - ("graph", graph.clone()), - ("result", result), - ("overwrite", request.getattr("overwrite")?), - ("adapter", context.getattr("adapter")?), - ("resolved_connection", context.getattr("connection_config")?), - ("cli_vars", request.getattr("cli_vars")?), - ], - ) { + let write: ContractWrite<'py> = ContractWrite { + project_dir: project_dir.clone(), + overwrite: request.getattr("overwrite")?, + adapter: context.getattr("adapter")?, + resolved_connection: context.getattr("connection_config")?, + cli_vars: request.getattr("cli_vars")?, + }; + let updated: Bound<'py, PyAny> = match write_contracts(&write, graph, &result) { Ok(updated) => updated, Err(error) => { if exception(py, &error) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/mod.rs index 2ef3aae6d5..7e5b22ae11 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod command; pub(crate) mod registration; +pub(crate) mod write; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs new file mode 100644 index 0000000000..b5f59917dd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs @@ -0,0 +1,195 @@ +//! Typed, metadata-preserving repository edits for contract adoption. + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const UTF8: &str = "utf-8"; + +/// The repository the edits apply to and how they apply. +pub(crate) struct ContractWrite<'py> { + pub(crate) project_dir: Bound<'py, PyAny>, + pub(crate) overwrite: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) resolved_connection: Bound<'py, PyAny>, + pub(crate) cli_vars: Bound<'py, PyAny>, +} + +/// Apply safe source-only edits atomically and validate the resulting project. +pub(crate) fn write_contracts<'py>( + write: &ContractWrite<'py>, + graph: &Bound<'py, PyAny>, + result: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = graph.py(); + let native = py.import(NATIVE_MODULE)?; + let project: Bound<'py, PyAny> = graph.getattr("project")?; + let models: Bound<'py, PyDict> = by_name(&project.getattr("models")?)?; + let sources: Bound<'py, PyDict> = by_name(&project.getattr("sources")?)?; + let model_type: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + let updates: Bound<'py, PyDict> = PyDict::new(py); + let updated_evidence: Bound<'py, PyList> = PyList::empty(py); + for evidence in result.getattr("evidence")?.try_iter()? { + let evidence: Bound<'py, PyAny> = evidence?; + let physical_columns: Bound<'py, PyAny> = evidence.getattr("physical_columns")?; + let findings: Bound<'py, PyAny> = evidence.getattr("findings")?; + if physical_columns.is_none() || !findings.is_truthy()? { + updated_evidence.append(evidence)?; + continue; + } + let name: Bound<'py, PyAny> = evidence.getattr("resource_name")?; + let (path, edited): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + if evidence.getattr("resource_type")?.eq(&model_type)? { + let model: Bound<'py, PyAny> = models.as_any().get_item(&name)?; + let edited: Bound<'py, PyAny> = call_with( + &native.getattr("edit_contract_model")?, + &[ + ("model", model.clone()), + ("physical_columns", physical_columns), + ("overwrite", write.overwrite.clone()), + ], + )?; + ( + write.project_dir.div(model.getattr("relative_path")?)?, + edited, + ) + } else { + let source: Bound<'py, PyAny> = sources.as_any().get_item(&name)?; + let path: Bound<'py, PyAny> = + source.getattr("source_file")?.getattr("file_path")?; + let edited: Bound<'py, PyAny> = call_with( + &native.getattr("edit_contract_source")?, + &[ + ("source", source), + ("physical_columns", physical_columns), + ("overwrite", write.overwrite.clone()), + ("contents", updates.as_any().call_method1("get", (&path,))?), + ], + )?; + (path, edited) + }; + let (contents, conflict): (Bound<'py, PyAny>, Bound<'py, PyAny>) = edited.extract()?; + if !contents.is_none() && contents.ne(read_text(&path)?)? { + updates.set_item(&path, contents)?; + } + let remaining: Bound<'py, PyAny> = call_with( + &native.getattr("remaining_contract_findings")?, + &[ + ("findings", findings), + ("overwrite", write.overwrite.clone()), + ("conflict", conflict), + ], + )?; + updated_evidence.append(replaced(&evidence, &[("findings", remaining)])?)?; + } + let evidence: Bound<'py, PyAny> = PyTuple::new(py, updated_evidence)?.into_any(); + if updates.is_empty() { + return replaced(result, &[("evidence", evidence)]); + } + let originals: Bound<'py, PyDict> = PyDict::new(py); + for path in updates.keys() { + originals.set_item(&path, read_text(&path)?)?; + } + if let Err(error) = applied(write, result, &updates) { + if error.is_instance_of::(py) { + write_all(&originals)?; + } + return Err(error); + } + let relative: Vec> = updates + .keys() + .iter() + .map(|path| path.call_method1("relative_to", (&write.project_dir,))) + .collect::>()?; + let written: Bound<'py, PyAny> = imported(py, "builtins", "sorted")?.call1((relative,))?; + replaced( + result, + &[ + ("evidence", evidence), + ( + "written_paths", + PyTuple::new(py, written.try_iter()?.collect::>>()?)?.into_any(), + ), + ], + ) +} + +/// Write every update, rediscover the project and compile it when an adapter is available. +fn applied<'py>( + write: &ContractWrite<'py>, + result: &Bound<'py, PyAny>, + updates: &Bound<'py, PyDict>, +) -> PyResult<()> { + let py: Python<'py> = result.py(); + let native = py.import(NATIVE_MODULE)?; + write_all(updates)?; + let discovered: Bound<'py, PyAny> = call_with( + &native.getattr("discover_project_inputs")?, + &[("project_dir", write.project_dir.clone())], + )?; + if write.adapter.is_none() { + return Ok(()); + } + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("no_cache", true)?; + let selection: Bound<'py, PyAny> = + imported(py, COMPILE_MODELS_MODULE, "CompileAnalysisSelection")? + .call((), Some(&keywords))?; + call_with( + &native.getattr("build_compiled_project")?, + &[ + ("discovered_inputs", discovered), + ("adapter", write.adapter.clone()), + ("selected_target", result.getattr("from_target")?), + ("resolved_connection", write.resolved_connection.clone()), + ("cli_vars", write.cli_vars.clone()), + ("analysis_selection", selection), + ], + )?; + Ok(()) +} + +/// Write each `path: contents` pair atomically, in order. +fn write_all(contents_by_path: &Bound<'_, PyDict>) -> PyResult<()> { + let py: Python<'_> = contents_by_path.py(); + let write_atomically: Bound<'_, PyAny> = imported(py, NATIVE_MODULE, "write_atomically")?; + for (path, contents) in contents_by_path.iter() { + call_with(&write_atomically, &[("path", path), ("contents", contents)])?; + } + Ok(()) +} + +fn read_text<'py>(path: &Bound<'py, PyAny>) -> PyResult> { + let keywords: Bound<'py, PyDict> = PyDict::new(path.py()); + keywords.set_item("encoding", UTF8)?; + path.call_method("read_text", (), Some(&keywords)) +} + +/// `{item.name: item for item in items}`. +fn by_name<'py>(items: &Bound<'py, PyAny>) -> PyResult> { + let named: Bound<'py, PyDict> = PyDict::new(items.py()); + for item in items.try_iter()? { + let item: Bound<'py, PyAny> = item?; + named.set_item(item.getattr("name")?, &item)?; + } + Ok(named) +} + +/// `dataclasses.replace(value, **fields)`. +fn replaced<'py>( + value: &Bound<'py, PyAny>, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let py: Python<'py> = value.py(); + let keywords: Bound<'py, PyDict> = PyDict::new(py); + for (name, field) in fields { + keywords.set_item(*name, field)?; + } + imported(py, "dataclasses", "replace")?.call((value,), Some(&keywords)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs index 15fce9daa5..274a060f52 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs @@ -19,7 +19,7 @@ const LOGGER_NAME: &str = "sqlbuild.cli.commands._helpers.cost.collection"; const COLLECTION_LOGGER_NAME: &str = "sqlbuild.cost"; const COST_TYPES: &str = "sqlbuild.cost.types"; const COST_MODELS: &str = "sqlbuild.cost.models"; -const COST_CONSTANTS: &str = "sqlbuild.cost.constants"; +const DEFAULT_RATE_SOURCE: &str = "default"; const CONFIGURED_RATE_SOURCE: &str = "configured"; const STATEMENT_LEDGER_PATH: [&str; 4] = ["target", "executions", "", "statements.jsonl"]; const BUILD_BREAKDOWN_LIMIT: usize = 5; @@ -273,7 +273,7 @@ pub(crate) fn finalize_build_cost<'py>( } let config: Bound<'py, PyAny> = finalization.getattr("config")?; let rate_source: String = if config.getattr("usd_per_credit_is_default")?.is_truthy()? { - imported(py, COST_CONSTANTS, "DEFAULT_RATE_SOURCE")?.extract()? + DEFAULT_RATE_SOURCE.to_owned() } else { CONFIGURED_RATE_SOURCE.to_owned() }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/exception_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/exception_types.rs new file mode 100644 index 0000000000..eaeadd2a00 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/exception_types.rs @@ -0,0 +1,19 @@ +//! Cost observability exceptions. + +use pyo3::create_exception; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +create_exception!( + sqlbuild._native, + CostArtifactError, + PyValueError, + "Raised when a persisted cost artifact has an invalid shape." +); + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add( + "CostArtifactError", + module.py().get_type::(), + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs index 242aaff1b3..f734d355cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod degradation; +pub(crate) mod exception_types; pub(crate) mod functions; pub(crate) mod records; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs index 4d9322f5fa..93da8e9177 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs @@ -9,7 +9,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; const COST_MODELS: &str = "sqlbuild.cost.models"; const COST_TYPES: &str = "sqlbuild.cost.types"; -const COST_EXCEPTIONS: &str = "sqlbuild.cost.exceptions"; +const COST_EXCEPTIONS: &str = "sqlbuild._native"; /// The `CostArtifactError` Python raised for the same artifact. pub(crate) fn artifact_error(py: Python<'_>, error: &CostArtifactError) -> PyErr { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/registration.rs index 7dbf137270..3b2f71866c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::cost_report::{degradation, functions}; +use crate::bindings::_helpers::cost_report::{degradation, exception_types, functions}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { degradation::register(module)?; + exception_types::register(module)?; functions::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs new file mode 100644 index 0000000000..99f5bc2578 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs @@ -0,0 +1,228 @@ +//! `allocate_run_cost`: price one run's share of observed Snowflake warehouse busy time. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cost_allocation::main::allocate_run_cost::allocate_run_cost as allocate; +use sqlbuild_core::cost_allocation::models::{ + AllocationStatus, CostQuery, PyDecimal, ResourceAllocation, RunAllocation, WarehouseKey, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::cost_telemetry::values::{ + COST_MODELS, COST_TYPES, instant_micros, native_decimal, python_decimal, +}; + +const PENDING_MESSAGE: &str = "Snowflake query history is not complete yet."; + +/// One `QueryCostObservation` as a native query, in the run when its run ID matches. +pub(crate) fn cost_query(observation: &Bound<'_, PyAny>, run_id: &str) -> PyResult { + let text = |name: &str| -> PyResult> { observation.getattr(name)?.extract() }; + Ok(CostQuery { + query_id: observation.getattr("query_id")?.extract()?, + warehouse: WarehouseKey { + name: observation.getattr("warehouse_name")?.extract()?, + size: observation.getattr("warehouse_size")?.extract()?, + kind: text("warehouse_type")?, + cluster: observation.getattr("cluster_number")?.extract()?, + }, + started_micros: instant_micros(&observation.getattr("started_at")?)?, + completed_micros: instant_micros(&observation.getattr("completed_at")?)?, + in_run: text("run_id")?.as_deref() == Some(run_id), + resource_type: text("resource_type")?, + resource_name: text("resource_name")?, + bytes_scanned: observation.getattr("bytes_scanned")?.extract()?, + }) +} + +/// The `CostStatus` member of a native allocation status. +pub(crate) fn status_member(status: AllocationStatus) -> &'static str { + match status { + AllocationStatus::Pending => "PENDING", + AllocationStatus::Partial => "PARTIAL", + AllocationStatus::Complete => "COMPLETE", + } +} + +/// Summary fields that collection derives on top of an allocation. +pub(crate) struct SummaryState { + pub(crate) status: &'static str, + pub(crate) message: Option, + pub(crate) limitations: Vec, + /// Expected, observed, missing and skipped statement counts. + pub(crate) counts: [usize; 4], +} + +fn resource_cost<'py>( + py: Python<'py>, + resource: &ResourceAllocation, +) -> PyResult> { + let decimal = |value: &PyDecimal| python_decimal(py, value); + call_with( + &imported(py, COST_MODELS, "ResourceCost")?, + &[ + ( + "resource_type", + resource.resource_type.clone().into_bound_py_any(py)?, + ), + ( + "resource_name", + resource.resource_name.clone().into_bound_py_any(py)?, + ), + ( + "warehouse_name", + resource.warehouse.name.clone().into_bound_py_any(py)?, + ), + ( + "warehouse_size", + resource.warehouse.size.clone().into_bound_py_any(py)?, + ), + ( + "warehouse_type", + resource.warehouse.kind.clone().into_bound_py_any(py)?, + ), + ( + "cluster_number", + resource.warehouse.cluster.into_bound_py_any(py)?, + ), + ("query_count", resource.query_count.into_bound_py_any(py)?), + ("attributed_seconds", decimal(&resource.attributed_seconds)?), + ( + "bytes_scanned", + resource.bytes_scanned.into_bound_py_any(py)?, + ), + ( + "estimated_compute_credits", + decimal(&resource.estimated_compute_credits)?, + ), + ("estimated_usd", decimal(&resource.estimated_usd)?), + ], + ) +} + +/// A `RunCostSummary`; without an allocation its totals keep their defaults. +pub(crate) fn summary_object<'py>( + allocation: Option<&RunAllocation>, + state: &SummaryState, + (usd_per_credit, rate_source): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = usd_per_credit.py(); + let [expected, observed, missing, skipped] = state.counts; + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ( + "status", + imported(py, COST_TYPES, "CostStatus")?.getattr(state.status)?, + ), + ("usd_per_credit", usd_per_credit.clone()), + ("rate_source", rate_source.clone()), + ("message", state.message.clone().into_bound_py_any(py)?), + ( + "limitations", + PyTuple::new(py, &state.limitations)?.into_any(), + ), + ("expected_statement_count", expected.into_bound_py_any(py)?), + ("observed_statement_count", observed.into_bound_py_any(py)?), + ("missing_statement_count", missing.into_bound_py_any(py)?), + ("skipped_statement_count", skipped.into_bound_py_any(py)?), + ]; + if let Some(allocation) = allocation { + let resources: Vec> = allocation + .resources + .iter() + .map(|resource| resource_cost(py, resource)) + .collect::>()?; + fields.extend([ + ("resources", PyTuple::new(py, resources)?.into_any()), + ("query_count", allocation.query_count.into_bound_py_any(py)?), + ( + "attributed_seconds", + python_decimal(py, &allocation.attributed_seconds)?, + ), + ( + "bytes_scanned", + allocation.bytes_scanned.into_bound_py_any(py)?, + ), + ( + "estimated_compute_credits", + python_decimal(py, &allocation.estimated_compute_credits)?, + ), + ( + "estimated_usd", + python_decimal(py, &allocation.estimated_usd)?, + ), + ]); + } + call_with(&imported(py, COST_MODELS, "RunCostSummary")?, &fields) +} + +/// The allocation that prices queries, or `None` while the run's history is pending. +pub(crate) fn priced(allocation: RunAllocation) -> (Option, SummaryState) { + let pending: bool = allocation.status == AllocationStatus::Pending; + let state: SummaryState = SummaryState { + status: status_member(allocation.status), + message: pending.then(|| PENDING_MESSAGE.to_owned()), + limitations: allocation.limitations.clone(), + counts: [0; 4], + }; + ((!pending).then_some(allocation), state) +} + +/// Native queries and the allocation of one run over observations. +pub(crate) fn allocate_observations( + observations: &[Bound<'_, PyAny>], + (run_id, usd_per_credit): (&str, &Bound<'_, PyAny>), + result_limit_reached: bool, +) -> PyResult { + let queries: Vec = observations + .iter() + .map(|observation| cost_query(observation, run_id)) + .collect::>()?; + Ok(allocate( + &queries, + &native_decimal(usd_per_credit)?, + result_limit_reached, + )) +} + +/// Concurrency-safe Snowflake busy-compute attribution for one run. +#[pyfunction] +#[pyo3(signature = (**options))] +fn allocate_run_cost<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "allocate_run_cost", + options, + ( + &["observations", "run_id", "usd_per_credit", "rate_source"], + &["result_limit_reached"], + ), + )?; + let observations: Vec> = arguments + .value("observations")? + .try_iter()? + .collect::>()?; + let run_id: String = arguments.value("run_id")?.extract()?; + let usd_per_credit: Bound<'py, PyAny> = arguments.value("usd_per_credit")?; + let allocation: RunAllocation = allocate_observations( + &observations, + (&run_id, &usd_per_credit), + arguments.value("result_limit_reached")?.is_truthy()?, + )?; + let (allocation, state) = priced(allocation); + summary_object( + allocation.as_ref(), + &state, + (&usd_per_credit, &arguments.value("rate_source")?), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(allocate_run_cost, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs new file mode 100644 index 0000000000..01c70906eb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs @@ -0,0 +1,189 @@ +//! `CostContext`: the per-resource cost attribution context, held in a Python `ContextVar`. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{IntoPyObjectExt, pyclass, pymethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::cost_telemetry::values::{COST_MODELS, process_object}; + +const CONTEXT_VARIABLE_NAME: &str = "sqlbuild_cost_resource_context"; +const DEFAULT_PHASE: &str = "execute"; +const DEFAULT_ATTEMPT: i64 = 1; + +static CURRENT: PyOnceLock> = PyOnceLock::new(); + +fn current_variable(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + process_object(&CURRENT, py, || { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("default", py.None())?; + imported(py, "contextvars", "ContextVar")?.call((CONTEXT_VARIABLE_NAME,), Some(&options)) + }) +} + +/// The active `CostResourceContext`, or `None`. +pub(crate) fn current_context(py: Python<'_>) -> PyResult> { + current_variable(py)?.call_method0("get") +} + +/// A `with` scope that makes one resource context current, or does nothing without one. +#[pyclass(module = "sqlbuild._native")] +pub(crate) struct CostScope { + context: Option>, + token: Option>, +} + +#[pymethods] +impl CostScope { + fn __enter__(&mut self, py: Python<'_>) -> PyResult<()> { + if let Some(context) = &self.context { + self.token = Some( + current_variable(py)? + .call_method1("set", (context.bind(py),))? + .unbind(), + ); + } + Ok(()) + } + + #[pyo3(signature = (*_exception))] + fn __exit__(&mut self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult<()> { + if let Some(token) = self.token.take() { + current_variable(py)?.call_method1("reset", (token.bind(py),))?; + } + Ok(()) + } +} + +fn or_current<'py>( + value: Bound<'py, PyAny>, + current: &Bound<'py, PyAny>, + field: &str, +) -> PyResult> { + if !value.is_none() || current.is_none() { + return Ok(value); + } + current.getattr(field) +} + +fn scope_of(context: Bound<'_, PyAny>) -> CostScope { + CostScope { + context: Some(context.unbind()), + token: None, + } +} + +/// Run-cost execution context entry points. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct CostContext; + +#[pymethods] +impl CostContext { + #[staticmethod] + fn current(py: Python<'_>) -> PyResult> { + current_context(py) + } + + #[staticmethod] + #[pyo3(signature = (**options))] + fn scope(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "scope", + options, + ( + &["run_id", "resource_type", "resource_name"], + &["ledger_path", "phase", "attempt", "on_statement_complete"], + ), + )?; + let current: Bound<'_, PyAny> = current_context(py)?; + let defaulted = |name: &str| -> PyResult> { + if arguments.contains(name)? { + return arguments.value(name); + } + match name { + "attempt" => DEFAULT_ATTEMPT.into_bound_py_any(py), + _ => DEFAULT_PHASE.into_bound_py_any(py), + } + }; + let context: Bound<'_, PyAny> = call_with( + &imported(py, COST_MODELS, "CostResourceContext")?, + &[ + ("run_id", arguments.value("run_id")?), + ("resource_type", arguments.value("resource_type")?), + ("resource_name", arguments.value("resource_name")?), + ( + "ledger_path", + or_current(arguments.value("ledger_path")?, ¤t, "ledger_path")?, + ), + ("phase", defaulted("phase")?), + ("attempt", defaulted("attempt")?), + ( + "on_statement_complete", + or_current( + arguments.value("on_statement_complete")?, + ¤t, + "on_statement_complete", + )?, + ), + ], + )?; + Ok(scope_of(context)) + } + + #[staticmethod] + #[pyo3(signature = (**options))] + fn resource_scope(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "resource_scope", + options, + (&["resource_type", "resource_name"], &["phase", "attempt"]), + )?; + let current: Bound<'_, PyAny> = current_context(py)?; + if current.is_none() { + return Ok(CostScope { + context: None, + token: None, + }); + } + let phase: Bound<'_, PyAny> = arguments.value("phase")?; + let attempt: Bound<'_, PyAny> = if arguments.contains("attempt")? { + arguments.value("attempt")? + } else { + DEFAULT_ATTEMPT.into_bound_py_any(py)? + }; + let context: Bound<'_, PyAny> = call_with( + &imported(py, COST_MODELS, "CostResourceContext")?, + &[ + ("run_id", current.getattr("run_id")?), + ("resource_type", arguments.value("resource_type")?), + ("resource_name", arguments.value("resource_name")?), + ("ledger_path", current.getattr("ledger_path")?), + ( + "phase", + if phase.is_none() { + current.getattr("phase")? + } else { + phase + }, + ), + ("attempt", attempt), + ( + "on_statement_complete", + current.getattr("on_statement_complete")?, + ), + ], + )?; + Ok(scope_of(context)) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs new file mode 100644 index 0000000000..55ccc7814e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs @@ -0,0 +1,74 @@ +//! Process-local statement-ledger failure tracking for the active build. + +use std::collections::HashMap; +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyTypeMethods; +use pyo3::{pyclass, pymethods}; + +use crate::bindings::_helpers::cost_telemetry::values::process_object; + +const HEALTH_ATTRIBUTE: &str = "COST_TELEMETRY_HEALTH"; + +static HEALTH: PyOnceLock> = PyOnceLock::new(); + +/// The process-wide ledger failure tracker, exposed as `COST_TELEMETRY_HEALTH`. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct CostTelemetryHealth { + ledger_failures: Mutex>, +} + +impl CostTelemetryHealth { + fn failures(&self) -> MutexGuard<'_, HashMap> { + self.ledger_failures + .lock() + .unwrap_or_else(PoisonError::into_inner) + } +} + +#[pymethods] +impl CostTelemetryHealth { + /// Remember the first ledger failure of `run_id` by its exception type name. + #[pyo3(signature = (*, run_id, error))] + fn mark_ledger_failure(&self, run_id: String, error: &Bound<'_, PyAny>) -> PyResult<()> { + let name: String = error.get_type().name()?.extract()?; + self.failures().entry(run_id).or_insert(name); + Ok(()) + } + + /// The ledger failure recorded for `run_id`, forgetting it. + #[pyo3(signature = (*, run_id))] + fn consume_ledger_failure(&self, run_id: &str) -> Option { + self.failures().remove(run_id) + } +} + +fn health(py: Python<'_>) -> PyResult> { + let tracker: &Bound<'_, PyAny> = process_object(&HEALTH, py, || { + Ok(Bound::new( + py, + CostTelemetryHealth { + ledger_failures: Mutex::new(HashMap::new()), + }, + )? + .into_any()) + })?; + Ok(tracker.cast::()?.clone()) +} + +/// Remember the first ledger failure of `run_id` by its exception type name. +pub(crate) fn mark_ledger_failure(run_id: String, error: &Bound<'_, PyAny>) -> PyResult<()> { + health(error.py())?.get().mark_ledger_failure(run_id, error) +} + +/// The ledger failure recorded for `run_id`, forgetting it. +pub(crate) fn consume_ledger_failure(py: Python<'_>, run_id: &str) -> PyResult> { + Ok(health(py)?.get().consume_ledger_failure(run_id)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add(HEALTH_ATTRIBUTE, health(module.py())?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs new file mode 100644 index 0000000000..25119a9b80 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs @@ -0,0 +1,261 @@ +//! The best-effort, append-only statement ledger that cost collection reads back. + +use std::fs::{self, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use pyo3::exceptions::{PyKeyError, PyTypeError, PyValueError}; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyBytes, PyDict, PyString, PyTuple, PyTypeMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, within, +}; +use crate::bindings::_helpers::cost_telemetry::health::mark_ledger_failure; +use crate::bindings::_helpers::cost_telemetry::values::{ + COST_EXCEPTIONS, COST_MODELS, process_object, +}; +use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; + +static LEDGER_LOCK: PyOnceLock> = PyOnceLock::new(); + +/// The process-wide `threading.Lock` that orders ledger appends and snapshot reads. +fn ledger_lock(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + process_object(&LEDGER_LOCK, py, || { + imported(py, "threading", "Lock")?.call0() + }) +} + +fn mark_failure(run_id: &Bound<'_, PyAny>, error: &Bound<'_, PyAny>) -> PyResult<()> { + mark_ledger_failure(run_id.extract()?, error) +} + +fn optional_text(value: Option) -> JsonValue { + value.map_or(JsonValue::Null, JsonValue::String) +} + +fn text(value: &Bound<'_, PyAny>) -> PyResult { + Ok(JsonValue::String(value.extract()?)) +} + +/// The ledger line of one statement: compact, key-sorted JSON. +fn ledger_line(arguments: &KeywordArguments<'_>, context: &Bound<'_, PyAny>) -> PyResult { + let query_id: Bound<'_, PyAny> = arguments.value("query_id")?; + let query_id: Option = + if query_id.is_instance_of::() && query_id.is_truthy()? { + Some(query_id.extract()?) + } else { + None + }; + let sql: String = arguments.value("sql")?.extract()?; + let error: Bound<'_, PyAny> = arguments.value("error")?; + let error_type: Option = if error.is_none() { + None + } else { + Some(error.get_type().name()?.extract()?) + }; + let attempt: String = context.getattr("attempt")?.extract::()?.to_string(); + let isoformat = |name: &str| -> PyResult { + text(&arguments.value(name)?.call_method0("isoformat")?) + }; + let mut fields: Vec<(String, JsonValue)> = vec![ + ("run_id".to_owned(), text(&context.getattr("run_id")?)?), + ( + "statement_id".to_owned(), + text(&arguments.value("statement_id")?)?, + ), + ( + "resource_type".to_owned(), + text(&context.getattr("resource_type")?)?, + ), + ( + "resource_name".to_owned(), + text(&context.getattr("resource_name")?)?, + ), + ("phase".to_owned(), text(&context.getattr("phase")?)?), + ( + "attempt".to_owned(), + JsonValue::Integer(JsonInteger::parse(&attempt).ok_or_else(|| { + PyValueError::new_err("statement ledger attempt must be an integer") + })?), + ), + ("query_id".to_owned(), optional_text(query_id)), + ("status".to_owned(), text(&arguments.value("status")?)?), + ("started_at".to_owned(), isoformat("started_at")?), + ("completed_at".to_owned(), isoformat("completed_at")?), + ( + "sql_sha256".to_owned(), + JsonValue::String(sha256_hex(sql.as_bytes())), + ), + ("error_type".to_owned(), optional_text(error_type)), + ]; + fields.sort_by(|left, right| left.0.cmp(&right.0)); + let options: StdlibJsonOptions = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }; + dumps(&JsonValue::Object(fields), &JsonDialect::Stdlib(options)) + .map_err(|error| PyValueError::new_err(format!("{error:?}"))) +} + +fn append_line(path: &Path, line: &str) -> std::io::Result<()> { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent)?; + } + let mut file = OpenOptions::new().append(true).create(true).open(path)?; + file.write_all(format!("{line}\n").as_bytes())?; + file.flush() +} + +/// Append statement identity without allowing telemetry I/O to affect execution. +#[pyfunction] +#[pyo3(signature = (**options))] +fn record_statement(py: Python<'_>, options: Option>) -> PyResult<()> { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "record_statement", + options, + ( + &[ + "context", + "statement_id", + "sql", + "query_id", + "status", + "started_at", + "completed_at", + ], + &["error"], + ), + )?; + let context: Bound<'_, PyAny> = arguments.value("context")?; + let path: Bound<'_, PyAny> = context.getattr("ledger_path")?; + if path.is_none() { + return Ok(()); + } + let appended: PyResult<()> = (|| { + let line: String = ledger_line(&arguments, &context)?; + let path: PathBuf = path.extract()?; + within(ledger_lock(py)?, |_| { + append_line(&path, &line).map_err(PyErr::from) + }) + })(); + match appended { + Ok(()) => Ok(()), + Err(error) if error.is_instance_of::(py) => { + mark_failure(&context.getattr("run_id")?, error.value(py).as_any()) + } + Err(error) => Err(error), + } +} + +/// One ledger line as a `StatementLedgerEntry`, raising the error Python's parsing raises. +fn ledger_entry<'py>(line: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = line.py(); + let payload: Bound<'py, PyAny> = imported(py, "json", "loads")?.call1((line,))?; + let query_id: Bound<'py, PyAny> = payload.call_method1("get", ("query_id",))?; + let field = |name: &str| -> PyResult> { + Ok(payload.get_item(name)?.str()?.into_any()) + }; + let datetime: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; + let parsed = |name: &str| -> PyResult> { + datetime.call_method1("fromisoformat", (field(name)?,)) + }; + let entry_fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("statement_id", field("statement_id")?), + ("run_id", field("run_id")?), + ("resource_type", field("resource_type")?), + ("resource_name", field("resource_name")?), + ("phase", field("phase")?), + ( + "attempt", + py.get_type::() + .call1((payload.get_item("attempt")?,))?, + ), + ( + "query_id", + if query_id.is_instance_of::() && query_id.is_truthy()? { + query_id + } else { + py.None().into_bound(py) + }, + ), + ("status", field("status")?), + ("started_at", parsed("started_at")?), + ("completed_at", parsed("completed_at")?), + ]; + call_with( + &imported(py, COST_MODELS, "StatementLedgerEntry")?, + &entry_fields, + ) +} + +fn entry_error(py: Python<'_>, error: &PyErr) -> bool { + error.is_instance_of::(py) + || error.is_instance_of::(py) + || error.is_instance_of::(py) +} + +/// Valid ledger entries of a stable snapshot, marking invalid lines and read failures. +pub(crate) fn read_ledger<'py>( + path: &Bound<'py, PyAny>, + run_id: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = path.py(); + if !path.call_method0("is_file")?.is_truthy()? { + return Ok(Vec::new()); + } + let native_path: PathBuf = path.extract()?; + let read: std::io::Result> = within(ledger_lock(py)?, |_| Ok(fs::read(&native_path)))?; + let bytes: Vec = match read { + Ok(bytes) => bytes, + Err(error) => { + mark_failure(run_id, PyErr::from(error).value(py).as_any())?; + return Ok(Vec::new()); + } + }; + let content: Bound<'py, PyAny> = PyBytes::new(py, &bytes).call_method1("decode", ("utf-8",))?; + let mut entries: Vec> = Vec::new(); + for line in content.call_method0("splitlines")?.try_iter()? { + match ledger_entry(&line?) { + Ok(entry) => entries.push(entry), + Err(error) if entry_error(py, &error) => { + let message: String = format!( + "invalid statement ledger entry ({})", + error.get_type(py).name()? + ); + let artifact_error: Bound<'py, PyAny> = + imported(py, COST_EXCEPTIONS, "CostArtifactError")?.call1((message,))?; + mark_failure(run_id, &artifact_error)?; + } + Err(error) => return Err(error), + } + } + Ok(entries) +} + +/// Read a stable snapshot of valid statement ledger entries. +#[pyfunction] +#[pyo3(signature = (*, path, run_id))] +fn read_statement_ledger<'py>( + path: &Bound<'py, PyAny>, + run_id: &Bound<'py, PyAny>, +) -> PyResult> { + PyTuple::new(path.py(), read_ledger(path, run_id)?) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(record_statement, module)?)?; + module.add_function(wrap_pyfunction!(read_statement_ledger, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs new file mode 100644 index 0000000000..89f46b212d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod allocation; +pub(crate) mod context; +pub(crate) mod health; +pub(crate) mod ledger; +pub(crate) mod registration; +pub(crate) mod snowflake; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs new file mode 100644 index 0000000000..4ce5de8966 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs @@ -0,0 +1,13 @@ +//! Register native run-cost allocation, Snowflake cost collection and statement telemetry. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::cost_telemetry::{allocation, context, health, ledger, snowflake}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + allocation::register(module)?; + context::register(module)?; + health::register(module)?; + ledger::register(module)?; + snowflake::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs new file mode 100644 index 0000000000..a9dd18e9be --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs @@ -0,0 +1,819 @@ +//! Snowflake query-history cost collection for one run, attributed through the statement ledger. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; + +use pyo3::exceptions::{PyTimeoutError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyList, PyListMethods, PyString, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::snowflake_cost::main::metadata_only_statement::metadata_only_statement; +use sqlbuild_analysis::snowflake_cost::main::sqlbuild_query_tag::sqlbuild_query_tag; +use sqlbuild_analysis::snowflake_cost::models::QueryTag; +use sqlbuild_core::cost_allocation::constants::QUERY_HISTORY_RESULT_LIMIT; +use sqlbuild_core::cost_allocation::main::query_history_sql::query_history_sql; +use sqlbuild_core::cost_allocation::models::{QueryHistoryWindow, RunAllocation}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::cost_telemetry::allocation::{ + SummaryState, allocate_observations, priced, summary_object, +}; +use crate::bindings::_helpers::cost_telemetry::health::consume_ledger_failure; +use crate::bindings::_helpers::cost_telemetry::ledger::read_ledger; +use crate::bindings::_helpers::cost_telemetry::values::COST_MODELS; +use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; + +const DEADLINE_SECONDS: f64 = 10.0; +const QUERY_TIMEOUT_SECONDS: &str = "5"; +const RUNNING_STATUS: &str = "RUNNING"; +const HOOK_RESOURCE_TYPE: &str = "hook"; +const ROW_WIDTH: usize = 14; +const NO_WAREHOUSE_COMPUTE: &str = "no_warehouse_compute"; +const MISSING_WAREHOUSE_METADATA: &str = "missing_warehouse_metadata"; +const INCOMPLETE_HISTORY: &str = "incomplete_history"; +const DEADLINE_MESSAGE: &str = "Snowflake cost collection deadline exceeded"; +const VISIBLE_QUERY_LIMITATION: &str = + "Concurrency sharing includes only queries visible to the executing Snowflake role."; +const RESULT_LIMIT_LIMITATION: &str = "Snowflake query-history result limit was reached."; +const LEDGER_FAILURE_PREFIX: &str = "Statement-ledger persistence failed ("; +const PENDING_STATUS: &str = "PENDING"; +const PARTIAL_STATUS: &str = "PARTIAL"; +const COMPLETE_STATUS: &str = "COMPLETE"; +const UNAVAILABLE_STATUS: &str = "UNAVAILABLE"; + +/// The connection, window inputs and deadline every query-history read shares. +struct History<'py> { + connection: Bound<'py, PyAny>, + database: Option, + classification_query_ids: Vec, + deadline: f64, + result_limit: usize, +} + +/// One query-history snapshot: observations, observed IDs, skip reasons and saturation. +struct Observed<'py> { + observations: Vec>, + observed: HashSet, + skipped: Vec<(String, &'static str)>, + saturated: bool, +} + +fn monotonic(py: Python<'_>) -> PyResult { + imported(py, "time", "monotonic")?.call0()?.extract() +} + +fn timedelta<'py>(py: Python<'py>, (unit, amount): (&str, i64)) -> PyResult> { + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item(unit, amount)?; + imported(py, DATETIME_MODULE, "timedelta")?.call((), Some(&options)) +} + +fn check_deadline(history: &History<'_>, py: Python<'_>) -> PyResult<()> { + if monotonic(py)? >= history.deadline { + return Err(PyTimeoutError::new_err(DEADLINE_MESSAGE)); + } + Ok(()) +} + +fn history_rows<'py>( + history: &History<'py>, + (start, end): (&Bound<'py, PyAny>, Option<&Bound<'py, PyAny>>), +) -> PyResult>> { + let py: Python<'py> = start.py(); + check_deadline(history, py)?; + let start_text: String = start.call_method0("isoformat")?.extract()?; + let end_text: Option = match end { + Some(end) => Some(end.call_method0("isoformat")?.extract()?), + None => None, + }; + let sql: String = query_history_sql(&QueryHistoryWindow { + database: history.database.as_deref(), + start: &start_text, + end: end_text.as_deref(), + classification_query_ids: &history.classification_query_ids, + result_limit: history.result_limit, + }); + let parameters: Bound<'py, PyDict> = PyDict::new(py); + parameters.set_item("STATEMENT_TIMEOUT_IN_SECONDS", QUERY_TIMEOUT_SECONDS)?; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("statement_params", parameters)?; + let cursor: Bound<'py, PyAny> = + history + .connection + .call_method("execute", (sql,), Some(&options))?; + check_deadline(history, py)?; + let rows: Vec> = cursor + .call_method0("fetchall")? + .try_iter()? + .map(|row| { + Ok(PyTuple::new(py, row?.try_iter()?.collect::>>()?)?.into_any()) + }) + .collect::>()?; + check_deadline(history, py)?; + Ok(rows) +} + +fn strings(values: &Bound<'_, PyAny>) -> PyResult> { + values.try_iter()?.map(|value| value?.extract()).collect() +} + +fn query_id(row: &Bound<'_, PyAny>) -> PyResult { + row.get_item(0)?.str()?.extract() +} + +/// Rows keyed by query ID, later rows replacing earlier ones, in query-ID order. +fn merged_rows<'py>( + rows: impl Iterator>, +) -> PyResult>> { + let mut by_query_id: BTreeMap> = BTreeMap::new(); + for row in rows { + by_query_id.insert(query_id(&row)?, row); + } + Ok(by_query_id.into_values().collect()) +} + +/// End-time-bounded rows, halving the window while a read reaches the result limit. +fn bounded_rows<'py>( + history: &History<'py>, + (start, end): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult<(Vec>, bool)> { + let py: Python<'py> = start.py(); + let rows: Vec> = history_rows(history, (start, Some(end)))?; + if rows.len() < history.result_limit { + return Ok((rows, false)); + } + let width: Bound<'py, PyAny> = end.sub(start)?; + if width.le(timedelta(py, ("milliseconds", 1))?)? { + return Ok((rows, true)); + } + let midpoint: Bound<'py, PyAny> = start.add(width.div(2)?)?; + let (left, left_saturated) = bounded_rows(history, (start, &midpoint))?; + let (right, right_saturated) = bounded_rows(history, (&midpoint, end))?; + Ok(( + merged_rows(left.into_iter().chain(right))?, + left_saturated || right_saturated, + )) +} + +fn running(end_time: &Bound<'_, PyAny>, status: &Bound<'_, PyAny>) -> PyResult { + Ok(end_time.is_none() || status.str()?.extract::()?.to_uppercase() == RUNNING_STATUS) +} + +fn unpacked<'py>(row: &Bound<'py, PyAny>) -> PyResult>> { + let fields: Vec> = row.try_iter()?.collect::>()?; + match fields.len() { + ROW_WIDTH => Ok(fields), + width if width > ROW_WIDTH => Err(PyValueError::new_err(format!( + "too many values to unpack (expected {ROW_WIDTH})" + ))), + width => Err(PyValueError::new_err(format!( + "not enough values to unpack (expected {ROW_WIDTH}, got {width})" + ))), + } +} + +fn python_int<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + let value: Bound<'py, PyAny> = if value.is_truthy()? { + value.clone() + } else { + 0_i64.into_bound_py_any(py)? + }; + py.get_type::().call1((value,)) +} + +fn optional_str<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(value.clone()); + } + Ok(value.str()?.into_any()) +} + +/// A history row as an observation, or the reason it carries no attributable compute. +fn row_observation<'py>( + row: &Bound<'py, PyAny>, + require_complete: bool, +) -> PyResult, &'static str>> { + let py: Python<'py> = row.py(); + let fields: Vec> = unpacked(row)?; + let [ + query_id, + query_tag, + warehouse_name, + warehouse_size, + warehouse_type, + cluster_number, + start_time, + end_time, + execution_ms, + bytes_scanned, + execution_status, + query_type, + query_text, + observed_at, + ] = <[Bound<'py, PyAny>; ROW_WIDTH]>::try_from(fields) + .map_err(|_| PyValueError::new_err("query history row width changed"))?; + let mut milliseconds: Bound<'py, PyAny> = python_int(&execution_ms)?; + if milliseconds.lt(0)? { + milliseconds = 0_i64.into_bound_py_any(py)?; + } + let query_type: String = if query_type.is_truthy()? { + query_type.str()?.extract()? + } else { + String::new() + }; + let query_text: Option = if query_text.is_instance_of::() { + Some(query_text.extract()?) + } else { + None + }; + if metadata_only_statement(&query_type, query_text.as_deref()) { + return Ok(Err(NO_WAREHOUSE_COMPUTE)); + } + let running: bool = running(&end_time, &execution_status)?; + if require_complete && running { + return Ok(Err(INCOMPLETE_HISTORY)); + } + if milliseconds.eq(0)? || warehouse_name.is_none() || warehouse_size.is_none() { + return Ok(Err(MISSING_WAREHOUSE_METADATA)); + } + let tag: Option = if query_tag.is_instance_of::() { + sqlbuild_query_tag(&query_tag.extract::()?) + } else { + None + }; + let effective_end: Bound<'py, PyAny> = if running { observed_at } else { end_time }; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("milliseconds", &milliseconds)?; + let busy_start: Bound<'py, PyAny> = + effective_end.sub(imported(py, DATETIME_MODULE, "timedelta")?.call((), Some(&options))?)?; + let active_start: Bound<'py, PyAny> = if busy_start.gt(&start_time)? { + busy_start + } else { + start_time + }; + if effective_end.le(&active_start)? { + return Ok(Err(NO_WAREHOUSE_COMPUTE)); + } + let tag: QueryTag = tag.unwrap_or_default(); + let observation: Bound<'py, PyAny> = call_with( + &imported(py, COST_MODELS, "QueryCostObservation")?, + &[ + ("query_id", query_id.str()?.into_any()), + ("warehouse_name", warehouse_name.str()?.into_any()), + ("warehouse_size", warehouse_size.str()?.into_any()), + ("warehouse_type", optional_str(&warehouse_type)?), + ( + "cluster_number", + if cluster_number.is_none() { + cluster_number + } else { + py.get_type::() + .call1((cluster_number,))? + }, + ), + ("started_at", active_start), + ("completed_at", effective_end), + ("execution_ms", milliseconds), + ("bytes_scanned", python_int(&bytes_scanned)?), + ("execution_status", execution_status.str()?.into_any()), + ("run_id", tag.run_id.into_bound_py_any(py)?), + ("resource_type", tag.resource_type.into_bound_py_any(py)?), + ("resource_name", tag.resource_name.into_bound_py_any(py)?), + ], + )?; + Ok(Ok(observation)) +} + +/// Observations of the run window: open-ended history, bounded windows when saturated. +fn query_observations<'py>( + history: &History<'py>, + (started_at, completed_at): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + expected: &HashSet, +) -> PyResult> { + let py: Python<'py> = started_at.py(); + let skew: Bound<'py, PyAny> = timedelta(py, ("minutes", 10))?; + let window_start: Bound<'py, PyAny> = started_at.sub(&skew)?; + let open_rows: Vec> = history_rows(history, (&window_start, None))?; + let mut saturated: bool = open_rows.len() >= history.result_limit; + let rows: Vec> = if saturated { + let (bounded, bounded_saturated) = + bounded_rows(history, (&window_start, &completed_at.add(&skew)?))?; + let mut still_running: Vec> = Vec::new(); + for row in open_rows { + if running(&row.get_item(7)?, &row.get_item(10)?)? { + still_running.push(row); + } + } + saturated = bounded_saturated; + merged_rows(still_running.into_iter().chain(bounded))? + } else { + open_rows + }; + let observations: Bound<'py, PyList> = PyList::empty(py); + let mut observed: HashSet = HashSet::new(); + let mut skipped: Vec<(String, &'static str)> = Vec::new(); + for row in rows { + let id: String = query_id(&row)?; + observed.insert(id.clone()); + match row_observation(&row, expected.contains(&id))? { + Ok(observation) => observations.append(observation)?, + Err(reason) => match skipped.iter_mut().find(|(skipped_id, _)| *skipped_id == id) { + Some(entry) => entry.1 = reason, + None => skipped.push((id, reason)), + }, + } + } + let key: Bound<'py, PyAny> = + imported(py, "operator", "attrgetter")?.call1(("started_at", "query_id"))?; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("key", key)?; + observations.call_method("sort", (), Some(&options))?; + Ok(Observed { + observations: observations.iter().collect(), + observed, + skipped, + saturated, + }) +} + +/// A collected summary: the allocation it prices, if any, and its derived fields. +struct Summary { + allocation: Option, + state: SummaryState, +} + +/// Sorted unique limitations after adding `added`. +fn with_limitations( + limitations: &[String], + added: impl IntoIterator, +) -> Vec { + let limitations: BTreeSet = limitations.iter().cloned().chain(added).collect(); + limitations.into_iter().collect() +} + +fn partial(message: &str) -> Summary { + Summary { + allocation: None, + state: SummaryState { + status: PARTIAL_STATUS, + message: Some(message.to_owned()), + limitations: Vec::new(), + counts: [0; 4], + }, + } +} + +fn with_ledger_failure(mut summary: Summary, failure: Option<&str>) -> Summary { + let Some(failure) = failure else { + return summary; + }; + summary.state.status = PARTIAL_STATUS; + summary.state.limitations = with_limitations( + &summary.state.limitations, + [format!("{LEDGER_FAILURE_PREFIX}{failure}).")], + ); + if summary.state.message.is_none() { + summary.state.message = Some("Cost telemetry was only partially persisted.".to_owned()); + } + summary +} + +/// Ledger expectations a summary is checked against. +struct Expected<'a> { + statement_count: usize, + query_ids: &'a HashSet, +} + +fn with_observation_gaps( + mut summary: Summary, + expected: &Expected<'_>, + observed: &Observed<'_>, +) -> Summary { + let relevant: Vec<&'static str> = observed + .skipped + .iter() + .filter(|(id, _)| expected.query_ids.contains(id)) + .map(|(_, reason)| *reason) + .collect(); + let missing_metadata: bool = relevant.contains(&MISSING_WAREHOUSE_METADATA); + let incomplete: bool = relevant.contains(&INCOMPLETE_HISTORY); + let mut added: Vec = Vec::new(); + if missing_metadata { + added.push( + "One or more run statements had missing Snowflake warehouse metadata.".to_owned(), + ); + } + if incomplete { + added.push("One or more run statements were still incomplete in query history.".to_owned()); + } + summary.state.limitations = with_limitations(&summary.state.limitations, added); + let query_count: usize = summary + .allocation + .as_ref() + .map_or(0, |allocation| allocation.query_count); + if missing_metadata || (incomplete && query_count > 0) { + summary.state.status = PARTIAL_STATUS; + } + let visible: usize = expected.query_ids.intersection(&observed.observed).count(); + summary.state.counts = [ + expected.statement_count, + visible, + expected.statement_count - visible, + relevant.len(), + ]; + summary +} + +fn incomplete_observation(expected: &HashSet, observed: &Observed<'_>) -> bool { + observed.skipped.iter().any(|(id, reason)| { + expected.contains(id) + && (*reason == MISSING_WAREHOUSE_METADATA || *reason == INCOMPLETE_HISTORY) + }) +} + +/// `replace(observation, run_id=..., resource_type=..., resource_name=...)` from the ledger entry. +fn ledger_attributed<'py>( + observation: &Bound<'py, PyAny>, + ledger: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = observation.py(); + let entry: Option> = + pyo3::types::PyDictMethods::get_item(ledger, observation.getattr("query_id")?)?; + let field = |name: &str| -> PyResult> { + match &entry { + Some(entry) => entry.getattr(name), + None => Ok(py.None().into_bound(py)), + } + }; + let changes: Bound<'py, PyDict> = PyDict::new(py); + changes.set_item("run_id", field("run_id")?)?; + changes.set_item("resource_type", field("resource_type")?)?; + changes.set_item("resource_name", field("resource_name")?)?; + imported(py, "dataclasses", "replace")?.call((observation,), Some(&changes)) +} + +/// Collection inputs read from the keyword arguments. +struct Collection<'py> { + run_id: Bound<'py, PyAny>, + started_at: Bound<'py, PyAny>, + completed_at: Bound<'py, PyAny>, + usd_per_credit: Bound<'py, PyAny>, + rate_source: Bound<'py, PyAny>, + attempts: i64, + retry_delay_seconds: f64, +} + +fn collect<'py>( + collection: &Collection<'py>, + mut history: History<'py>, + ledger_path: &Bound<'py, PyAny>, +) -> PyResult { + let py: Python<'py> = ledger_path.py(); + history.deadline = monotonic(py)? + DEADLINE_SECONDS; + let run_id: String = collection.run_id.extract()?; + let ledger: Vec> = read_ledger(ledger_path, &collection.run_id)?; + let failure: Option = consume_ledger_failure(py, &run_id)?; + let by_query_id: Bound<'py, PyDict> = PyDict::new(py); + for entry in &ledger { + let id: Bound<'py, PyAny> = entry.getattr("query_id")?; + if !id.is_none() { + by_query_id.set_item(id, entry)?; + } + } + let mut expected_ids: HashSet = HashSet::new(); + let mut classification: Vec = Vec::new(); + for (id, entry) in pyo3::types::PyDictMethods::iter(&by_query_id) { + let id: String = id.extract()?; + if !entry.getattr("resource_type")?.eq(HOOK_RESOURCE_TYPE)? { + classification.push(id.clone()); + } + expected_ids.insert(id); + } + history.classification_query_ids = classification; + if !ledger.is_empty() && expected_ids.is_empty() { + let mut summary: Summary = with_ledger_failure( + partial("No Snowflake query IDs were available in the statement ledger."), + failure.as_deref(), + ); + summary.state.counts = [ledger.len(), 0, ledger.len(), 0]; + return Ok(summary); + } + let expected: Expected<'_> = Expected { + statement_count: ledger.len(), + query_ids: &expected_ids, + }; + let mut summary: Option = None; + let mut observed: Observed<'py> = Observed { + observations: Vec::new(), + observed: HashSet::new(), + skipped: Vec::new(), + saturated: false, + }; + for attempt in 0..collection.attempts { + observed = query_observations( + &history, + (&collection.started_at, &collection.completed_at), + &expected_ids, + )?; + let attributed: Vec> = observed + .observations + .iter() + .map(|observation| ledger_attributed(observation, &by_query_id)) + .collect::>()?; + let allocation: RunAllocation = allocate_observations( + &attributed, + (&run_id, &collection.usd_per_credit), + observed.saturated, + )?; + let (allocation, mut state) = priced(allocation); + state.limitations = + with_limitations(&state.limitations, [VISIBLE_QUERY_LIMITATION.to_owned()]); + let current: Summary = Summary { allocation, state }; + if expected_ids.is_subset(&observed.observed) + && !incomplete_observation(&expected_ids, &observed) + { + let mut complete: Summary = with_observation_gaps(current, &expected, &observed); + if complete.state.status == PENDING_STATUS { + complete.state.status = COMPLETE_STATUS; + complete.state.message = + Some("No cost-bearing Snowflake warehouse queries were observed.".to_owned()); + } + return Ok(with_ledger_failure(complete, failure.as_deref())); + } + summary = Some(current); + if attempt + 1 < collection.attempts { + if history.deadline - monotonic(py)? <= collection.retry_delay_seconds { + break; + } + imported(py, "time", "sleep")?.call1((collection.retry_delay_seconds,))?; + } + } + let mut summary: Summary = + summary.unwrap_or_else(|| partial("Snowflake query history could not be collected.")); + let missing: usize = expected_ids.difference(&observed.observed).count(); + let mut added: Vec = Vec::new(); + if missing > 0 { + added.push(format!( + "{missing} statement ledger query ID(s) were not visible in query history." + )); + } + if observed.saturated { + added.push(RESULT_LIMIT_LIMITATION.to_owned()); + } + summary.state.limitations = with_limitations(&summary.state.limitations, added); + let visible: bool = expected_ids + .intersection(&observed.observed) + .next() + .is_some(); + let now: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")? + .call_method1("now", (imported(py, DATETIME_MODULE, "UTC")?,))?; + let expired: bool = now + .sub(&collection.completed_at)? + .ge(timedelta(py, ("days", 7))?)?; + if expired && !visible { + summary.state.status = UNAVAILABLE_STATUS; + summary.state.message = + Some("Snowflake query history is no longer available for this run.".to_owned()); + } else if !visible && !observed.saturated { + summary.state.status = PENDING_STATUS; + summary.state.message = Some( + "Snowflake query history is not complete yet. Run `sqb cost latest` to refresh." + .to_owned(), + ); + } else { + summary.state.status = PARTIAL_STATUS; + if summary.state.message.is_none() { + summary.state.message = + Some("Snowflake query history was only partially available.".to_owned()); + } + } + let summary: Summary = with_observation_gaps(summary, &expected, &observed); + Ok(with_ledger_failure(summary, failure.as_deref())) +} + +/// Collect visible history and attribute exact ledger query IDs to one run. +#[pyfunction] +#[pyo3(signature = (**options))] +fn collect_snowflake_cost<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "collect_snowflake_cost", + options, + ( + &[ + "connection", + "run_id", + "started_at", + "completed_at", + "statement_ledger_path", + "usd_per_credit", + "rate_source", + ], + &[ + "database", + "attempts", + "retry_delay_seconds", + "result_limit", + ], + ), + )?; + let optional = |name: &str, default: Bound<'py, PyAny>| -> PyResult> { + Ok(if arguments.contains(name)? { + arguments.value(name)? + } else { + default + }) + }; + let collection: Collection<'py> = Collection { + run_id: arguments.value("run_id")?, + started_at: arguments.value("started_at")?, + completed_at: arguments.value("completed_at")?, + usd_per_credit: arguments.value("usd_per_credit")?, + rate_source: arguments.value("rate_source")?, + attempts: optional("attempts", 3_i64.into_bound_py_any(py)?)?.extract()?, + retry_delay_seconds: optional("retry_delay_seconds", 1.0_f64.into_bound_py_any(py)?)? + .extract()?, + }; + let history: History<'py> = History { + connection: arguments.value("connection")?, + database: arguments.value("database")?.extract()?, + classification_query_ids: Vec::new(), + deadline: 0.0, + result_limit: optional( + "result_limit", + QUERY_HISTORY_RESULT_LIMIT.into_bound_py_any(py)?, + )? + .extract()?, + }; + let summary: Summary = collect( + &collection, + history, + &arguments.value("statement_ledger_path")?, + )?; + summary_object( + summary.allocation.as_ref(), + &summary.state, + (&collection.usd_per_credit, &collection.rate_source), + ) +} + +/// Query-history observations of one window, as cost collection reads them. +#[pyfunction] +#[pyo3(signature = (**options))] +fn snowflake_query_observations<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "snowflake_query_observations", + options, + ( + &[ + "connection", + "started_at", + "completed_at", + "expected_query_ids", + "classification_query_ids", + "deadline", + ], + &["database", "result_limit"], + ), + )?; + let result_limit: Bound<'py, PyAny> = arguments.value("result_limit")?; + let history: History<'py> = History { + connection: arguments.value("connection")?, + database: arguments.value("database")?.extract()?, + classification_query_ids: strings(&arguments.value("classification_query_ids")?)?, + deadline: arguments.value("deadline")?.extract()?, + result_limit: if result_limit.is_none() { + QUERY_HISTORY_RESULT_LIMIT + } else { + result_limit.extract()? + }, + }; + let expected: HashSet = strings(&arguments.value("expected_query_ids")?)? + .into_iter() + .collect(); + let observed: Observed<'py> = query_observations( + &history, + ( + &arguments.value("started_at")?, + &arguments.value("completed_at")?, + ), + &expected, + )?; + let skipped: Bound<'py, PyDict> = PyDict::new(py); + for (id, reason) in &observed.skipped { + skipped.set_item(id, reason)?; + } + PyTuple::new( + py, + [ + PyTuple::new(py, &observed.observations)?.into_any(), + pyo3::types::PyFrozenSet::new(py, &observed.observed)?.into_any(), + skipped.into_any(), + observed.saturated.into_bound_py_any(py)?, + ], + ) +} + +/// The query-history SQL of one window; an absent end reads every query ending after the start. +#[pyfunction] +#[pyo3(signature = (**options))] +fn render_snowflake_query_history_sql( + py: Python<'_>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "render_snowflake_query_history_sql", + options, + ( + &["started_at", "classification_query_ids"], + &["database", "completed_at", "result_limit"], + ), + )?; + let start: String = arguments + .value("started_at")? + .call_method0("isoformat")? + .extract()?; + let completed_at: Bound<'_, PyAny> = arguments.value("completed_at")?; + let end: Option = if completed_at.is_none() { + None + } else { + Some(completed_at.call_method0("isoformat")?.extract()?) + }; + let result_limit: Bound<'_, PyAny> = arguments.value("result_limit")?; + let database: Option = arguments.value("database")?.extract()?; + Ok(query_history_sql(&QueryHistoryWindow { + database: database.as_deref(), + start: &start, + end: end.as_deref(), + classification_query_ids: &strings(&arguments.value("classification_query_ids")?)?, + result_limit: if result_limit.is_none() { + QUERY_HISTORY_RESULT_LIMIT + } else { + result_limit.extract()? + }, + })) +} + +/// Whether Snowflake runs a statement of this type and text without warehouse compute. +#[pyfunction] +#[pyo3(signature = (*, query_type, query_text))] +fn snowflake_statement_is_metadata_only( + query_type: &Bound<'_, PyAny>, + query_text: &Bound<'_, PyAny>, +) -> PyResult { + let query_type: String = if query_type.is_truthy()? { + query_type.str()?.extract()? + } else { + String::new() + }; + let query_text: Option = if query_text.is_instance_of::() { + Some(query_text.extract()?) + } else { + None + }; + Ok(metadata_only_statement(&query_type, query_text.as_deref())) +} + +/// The run attribution of a SQLBuild query tag, or `None` for any other tag. +#[pyfunction] +fn parse_sqlbuild_query_tag<'py>( + value: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = value.py(); + if !value.is_instance_of::() { + return Ok(None); + } + let Some(tag) = sqlbuild_query_tag(&value.extract::()?) else { + return Ok(None); + }; + let attribution: Bound<'py, PyDict> = PyDict::new(py); + attribution.set_item("run_id", tag.run_id)?; + attribution.set_item("resource_type", tag.resource_type)?; + attribution.set_item("resource_name", tag.resource_name)?; + Ok(Some(attribution)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(collect_snowflake_cost, module)?)?; + module.add_function(wrap_pyfunction!(snowflake_query_observations, module)?)?; + module.add_function(wrap_pyfunction!( + render_snowflake_query_history_sql, + module + )?)?; + module.add_function(wrap_pyfunction!( + snowflake_statement_is_metadata_only, + module + )?)?; + module.add_function(wrap_pyfunction!(parse_sqlbuild_query_tag, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs new file mode 100644 index 0000000000..30d073aa86 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs @@ -0,0 +1,80 @@ +//! Python `Decimal` and `datetime` values as exact native decimals and microsecond instants. + +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_core::cost_allocation::models::PyDecimal; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; +use crate::bindings::_helpers::state_rows::timestamps::offset_aware; + +pub(crate) const COST_MODELS: &str = "sqlbuild.cost.models"; +pub(crate) const COST_TYPES: &str = "sqlbuild.cost.types"; +pub(crate) const COST_EXCEPTIONS: &str = "sqlbuild._native"; +const DECIMAL_MODULE: &str = "decimal"; + +/// A finite non-negative `Decimal` as its exact digits and exponent. +pub(crate) fn native_decimal(value: &Bound<'_, PyAny>) -> PyResult { + let (sign, digits, exponent): (u8, Vec, Bound<'_, PyAny>) = + value.call_method0("as_tuple")?.extract()?; + let Ok(exponent) = exponent.extract::() else { + return Err(PyValueError::new_err("cost decimals must be finite")); + }; + if sign != 0 && digits.iter().any(|digit| *digit != 0) { + return Err(PyValueError::new_err("cost decimals must not be negative")); + } + let leading: usize = digits.iter().take_while(|digit| **digit == 0).count(); + let digits: Vec = if leading == digits.len() { + vec![0] + } else { + digits[leading..].to_vec() + }; + Ok(PyDecimal { digits, exponent }) +} + +/// `Decimal((0, digits, exponent))`. +pub(crate) fn python_decimal<'py>( + py: Python<'py>, + value: &PyDecimal, +) -> PyResult> { + let parts: Bound<'py, PyTuple> = PyTuple::new( + py, + [ + 0_i64.into_bound_py_any(py)?, + PyTuple::new(py, &value.digits)?.into_any(), + value.exponent.into_bound_py_any(py)?, + ], + )?; + imported(py, DECIMAL_MODULE, "Decimal")?.call1((parts,)) +} + +/// Microseconds since the epoch: UTC for aware values, the wall clock for naive ones. +pub(crate) fn instant_micros(value: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = value.py(); + let datetime: Bound<'_, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; + let epoch: Bound<'_, PyAny> = if offset_aware(value)? { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("tzinfo", imported(py, DATETIME_MODULE, "UTC")?)?; + datetime.call((1970, 1, 1), Some(&options))? + } else { + datetime.call1((1970, 1, 1))? + }; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("microseconds", 1)?; + let micro: Bound<'_, PyAny> = + imported(py, DATETIME_MODULE, "timedelta")?.call((), Some(&options))?; + value.sub(epoch)?.floor_div(micro)?.extract() +} + +/// The process-wide Python object in `cell`, built by `build` on first use. +pub(crate) fn process_object<'py>( + cell: &'static PyOnceLock>, + py: Python<'py>, + build: impl FnOnce() -> PyResult>, +) -> PyResult<&'py Bound<'py, PyAny>> { + cell.get_or_try_init(py, || Ok::, PyErr>(build()?.unbind())) + .map(|object| object.bind(py)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index 7e9b95a669..570b82d1fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -8,6 +8,7 @@ use crate::bindings::_helpers::build_command::support::{boolean, keyword_relay, use crate::bindings::_helpers::command_runtime::connection::{ anchor_duckdb_database, with_dbt_profile, }; +use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; @@ -120,18 +121,7 @@ fn open_connection<'py>( )); } let relayed = call_with(&imported(py, HOOKS, "ConnectionHooks")?, &pairs)?; - call_with( - &imported( - py, - "sqlbuild.runtime.contracts.main.open_connection", - "open_connection_with_hooks", - )?, - &[ - ("adapter", environment.getattr("adapter")?), - ("connection_config", config), - ("hooks", relayed), - ], - ) + connection_with_hooks(&environment.getattr("adapter")?, &config, &relayed) } /// Build the SQLBuild plan from current warehouse facts, closing the custom adapter connection. @@ -193,11 +183,7 @@ pub(crate) fn build_sqlbuild_plan_output<'py>( )?; let discovered = environment.getattr("discovered_inputs")?; let result = call_with( - &imported( - py, - "sqlbuild.compiler.planner.main.execution.execution", - "build_execution_plan", - )?, + &imported(py, "sqlbuild._native", "build_execution_plan")?, &[ ("project", environment.getattr("project")?), ("adapter", environment.getattr("adapter")?), @@ -226,11 +212,7 @@ pub(crate) fn build_sqlbuild_plan_output<'py>( ) => { call_with( - &imported( - py, - "sqlbuild.compiler.planner.main.execution.display_plan", - "build_display_only_sqlbuild_plan", - )?, + &imported(py, "sqlbuild._native", "build_display_only_sqlbuild_plan")?, &[ ("project", environment.getattr("project")?), ("selected_model_names", selected), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs index 75905f28f0..c62b84ea12 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs @@ -108,7 +108,7 @@ fn execute_dbt_commands<'py>( let argv = arguments.value("merged_argv")?; let output = arguments.value("progress_stream")?; let style = call_with( - &imported(py, "sqlbuild.presentation.classes.cli_style", "CliStyle")?, + &imported(py, "sqlbuild._native", "CliStyle")?, &[("use_color", arguments.value("use_color")?)], )?; let result_type = imported( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs index 2c82dfb7d8..53267ead8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs @@ -2,7 +2,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_core::build_command::constants::{ + EXECUTION_LIMIT_DBT_NOTE, MISSING_ORIGIN_DBT_HELP, RETENTION_DECREASE_DBT_HELP, + SNAPSHOT_FULL_REFRESH_DBT_HELP, TABLE_TYPE_DOWNGRADE_DBT_HELP, +}; use crate::bindings::_helpers::build_command::support::{ boolean, emit, joined, keyword_relay, none, text, tuple, @@ -24,7 +28,6 @@ const WORK_OPTIONS: (&[&str], &[&str]) = ( ); const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const COMMAND_CONSTANTS: &str = "sqlbuild.cli.commands.constants"; /// Interop has no allow flags; destructive work uses the ordinary interactive safety gates. #[pyfunction] @@ -38,23 +41,23 @@ fn dbt_interop_plan_safety_gates(py: Python<'_>) -> PyResult> { ("allow_retention_decrease", boolean(py, false)), ( "missing_origin_help", - imported(py, COMMAND_CONSTANTS, "MISSING_ORIGIN_DBT_HELP")?, + MISSING_ORIGIN_DBT_HELP.into_bound_py_any(py)?, ), ( "snapshot_full_refresh_help", - imported(py, COMMAND_CONSTANTS, "SNAPSHOT_FULL_REFRESH_DBT_HELP")?, + SNAPSHOT_FULL_REFRESH_DBT_HELP.into_bound_py_any(py)?, ), ( "table_type_downgrade_help", - imported(py, COMMAND_CONSTANTS, "TABLE_TYPE_DOWNGRADE_DBT_HELP")?, + TABLE_TYPE_DOWNGRADE_DBT_HELP.into_bound_py_any(py)?, ), ( "retention_decrease_help", - imported(py, COMMAND_CONSTANTS, "RETENTION_DECREASE_DBT_HELP")?, + RETENTION_DECREASE_DBT_HELP.into_bound_py_any(py)?, ), ( "execution_limit_note", - imported(py, COMMAND_CONSTANTS, "EXECUTION_LIMIT_DBT_NOTE")?, + EXECUTION_LIMIT_DBT_NOTE.into_bound_py_any(py)?, ), ], ) @@ -113,7 +116,7 @@ fn execute_dbt_sqlbuild_work<'py>( ], )?; let style = call_with( - &imported(py, "sqlbuild.presentation.classes.cli_style", "CliStyle")?, + &imported(py, "sqlbuild._native", "CliStyle")?, &[("use_color", color.clone())], )?; emit( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs index 5dfdec4811..14fc125d3d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs @@ -1 +1,2 @@ pub(crate) mod redaction; +pub(crate) mod trackers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs new file mode 100644 index 0000000000..33f860fdac --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs @@ -0,0 +1,212 @@ +//! Invocation-scoped partial build phase timings and process resource deltas. + +use std::sync::{Mutex, PoisonError}; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyType}; +use pyo3::{pyclass, pymethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::{context_variable, imported}; + +const DIAGNOSTICS_MODELS_MODULE: &str = "sqlbuild.diagnostics.models"; +const PROCESS_RESOURCES_MODULE: &str = "sqlbuild.diagnostics._helpers.process_resources"; +const TRACKER_VARIABLE: &str = "sqlbuild_build_phase_timing_tracker"; +const PHASES: [&str; 6] = [ + "compile_seconds", + "planning_seconds", + "connection_preparation_seconds", + "schema_preparation_seconds", + "execution_seconds", + "cost_collection_seconds", +]; + +static CURRENT_TRACKER: PyOnceLock> = PyOnceLock::new(); + +/// The context variable holding the active build timing tracker. +fn current_variable(py: Python<'_>) -> PyResult> { + context_variable(py, &CURRENT_TRACKER, TRACKER_VARIABLE) +} + +/// `monotonic`, or `time.monotonic` when none is given. +fn clock<'py>( + py: Python<'py>, + monotonic: Option>, +) -> PyResult> { + match monotonic { + Some(monotonic) => Ok(monotonic), + None => imported(py, "time", "monotonic"), + } +} + +/// `max(0.0, later - earlier)`. +fn non_negative(later: f64, earlier: f64) -> f64 { + (later - earlier).max(0.0) +} + +/// Retain disjoint monotonic phase durations across exceptional exits. +#[pyclass(module = "sqlbuild._native")] +pub(crate) struct BuildPhaseTimingTracker { + monotonic: Py, + started_at: f64, + #[pyo3(get, set)] + compile_seconds: Option, + #[pyo3(get, set)] + planning_seconds: Option, + #[pyo3(get, set)] + connection_preparation_seconds: Option, + #[pyo3(get, set)] + schema_preparation_seconds: Option, + #[pyo3(get, set)] + execution_seconds: Option, + #[pyo3(get, set)] + cost_collection_seconds: Option, + tokens: Mutex>>, +} + +#[pymethods] +impl BuildPhaseTimingTracker { + #[new] + #[pyo3(signature = (*, monotonic = None))] + fn new(py: Python<'_>, monotonic: Option>) -> PyResult { + let monotonic: Bound<'_, PyAny> = clock(py, monotonic)?; + let started_at: f64 = monotonic.call0()?.extract()?; + Ok(Self { + monotonic: monotonic.unbind(), + started_at, + compile_seconds: None, + planning_seconds: None, + connection_preparation_seconds: None, + schema_preparation_seconds: None, + execution_seconds: None, + cost_collection_seconds: None, + tokens: Mutex::new(Vec::new()), + }) + } + + /// Return currently available phase durations and total wall time. + fn snapshot<'py>(&self, py: Python<'py>) -> PyResult> { + let now: f64 = self.monotonic.bind(py).call0()?.extract()?; + let values: [Option; 6] = [ + self.compile_seconds, + self.planning_seconds, + self.connection_preparation_seconds, + self.schema_preparation_seconds, + self.execution_seconds, + self.cost_collection_seconds, + ]; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + for (phase, value) in PHASES.into_iter().zip(values) { + keywords.set_item(phase, value)?; + } + keywords.set_item("total_seconds", non_negative(now, self.started_at))?; + imported(py, DIAGNOSTICS_MODELS_MODULE, "PartialBuildPhaseTimings")? + .call((), Some(&keywords)) + } + + /// Return the active build timing tracker. + #[classmethod] + fn current<'py>(cls: &Bound<'py, PyType>) -> PyResult> { + current_variable(cls.py())?.call_method0("get") + } + + /// Install this tracker for one command invocation and reset it afterward. + fn scope(slf: &Bound<'_, Self>) -> PhaseTimingScope { + PhaseTimingScope { + tracker: slf.clone().unbind(), + } + } +} + +impl BuildPhaseTimingTracker { + fn tokens(&self) -> std::sync::MutexGuard<'_, Vec>> { + self.tokens.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +/// Makes one tracker the active build timing tracker while entered. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct PhaseTimingScope { + tracker: Py, +} + +#[pymethods] +impl PhaseTimingScope { + fn __enter__(&self, py: Python<'_>) -> PyResult<()> { + let token: Bound<'_, PyAny> = + current_variable(py)?.call_method1("set", (self.tracker.bind(py),))?; + self.tracker.bind(py).borrow().tokens().push(token.unbind()); + Ok(()) + } + + #[pyo3(signature = (*_exception))] + fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult { + let token: Option> = self.tracker.bind(py).borrow().tokens().pop(); + if let Some(token) = token { + current_variable(py)?.call_method1("reset", (token,))?; + } + Ok(false) + } +} + +/// Capture wall and process resource deltas around one command. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct ProcessResourceTracker { + monotonic: Py, + resource_reader: Py, + started_at: f64, + started_user: f64, + started_system: f64, +} + +#[pymethods] +impl ProcessResourceTracker { + #[new] + #[pyo3(signature = (*, monotonic = None, resource_reader = None))] + fn new( + py: Python<'_>, + monotonic: Option>, + resource_reader: Option>, + ) -> PyResult { + let monotonic: Bound<'_, PyAny> = clock(py, monotonic)?; + let resource_reader: Bound<'_, PyAny> = match resource_reader { + Some(reader) => reader, + None => imported(py, PROCESS_RESOURCES_MODULE, "read_process_resources")?, + }; + let started_at: f64 = monotonic.call0()?.extract()?; + let (started_user, started_system, _): (f64, f64, Bound<'_, PyAny>) = + resource_reader.call0()?.extract()?; + Ok(Self { + monotonic: monotonic.unbind(), + resource_reader: resource_reader.unbind(), + started_at, + started_user, + started_system, + }) + } + + /// Return elapsed wall/CPU deltas and process peak RSS. + fn finish<'py>(&self, py: Python<'py>) -> PyResult> { + let (user_cpu, system_cpu, max_rss_bytes): (f64, f64, Bound<'py, PyAny>) = + self.resource_reader.bind(py).call0()?.extract()?; + let now: f64 = self.monotonic.bind(py).call0()?.extract()?; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("wall_seconds", non_negative(now, self.started_at))?; + keywords.set_item( + "user_cpu_seconds", + non_negative(user_cpu, self.started_user), + )?; + keywords.set_item( + "system_cpu_seconds", + non_negative(system_cpu, self.started_system), + )?; + keywords.set_item("max_rss_bytes", max_rss_bytes)?; + imported(py, DIAGNOSTICS_MODELS_MODULE, "ProcessResourceUsage")?.call((), Some(&keywords)) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs index 7c94d567db..5ed0391081 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs @@ -1,11 +1,17 @@ //! Setting guidance shown by diagnostics that depend on project settings. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyBoolMethods, PyDict, PyInt, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_config::project::main::explicit_references_help::explicit_references_help as references_help; use sqlbuild_config::project::main::microbatch_guidance::microbatch_guidance as concurrency_guidance; +use sqlbuild_config::project::main::setting_help_text::setting_help_text; +use sqlbuild_config::project::main::snippet_help_text::snippet_help_text; use sqlbuild_config::project::main::sql_analysis_off_guidance::sql_analysis_off_guidance as analysis_off; +use sqlbuild_config::project::models::{SettingSite, SettingText}; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; const SQL_ANALYSIS_KEY: &str = "sql_analysis"; /// The note and help for turning SQL analysis on, or `None` when it is effectively on. @@ -52,7 +58,56 @@ fn explicit_references_help(allowed: &str) -> String { references_help(allowed) } +/// A Python `bool | int | str` setting value as help text shows it. +fn setting_text(value: &Bound<'_, PyAny>) -> PyResult { + if let Ok(flag) = value.cast::() { + return Ok(SettingText::Bool(flag.is_true())); + } + if value.is_instance_of::() { + return Ok(SettingText::Integer(value.str()?.extract()?)); + } + if value.is_instance_of::() { + return Ok(SettingText::Text(value.extract()?)); + } + Err(PyTypeError::new_err( + "setting value must be a bool, int or str", + )) +} + +/// Render `, set this in :` followed by the exact TOML section and key. +#[pyfunction] +#[pyo3(signature = (**options))] +fn setting_help<'py>(py: Python<'py>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "setting_help", + options, + (&["purpose", "file_name", "section", "key", "value"], &[]), + )?; + let text = |name: &str| -> PyResult { arguments.value(name)?.extract() }; + let (file_name, section, key): (String, String, String) = + (text("file_name")?, text("section")?, text("key")?); + Ok(setting_help_text( + &text("purpose")?, + SettingSite { + file_name: &file_name, + section: §ion, + key: &key, + }, + &setting_text(&arguments.value("value")?)?, + )) +} + +/// Render `, :` followed by the exact lines to write, one per line. +#[pyfunction] +#[pyo3(signature = (*, purpose, target, lines))] +fn snippet_help(purpose: &str, target: &str, lines: Vec) -> String { + snippet_help_text(purpose, target, &lines) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(setting_help, module)?)?; + module.add_function(wrap_pyfunction!(snippet_help, module)?)?; module.add_function(wrap_pyfunction!(sql_analysis_off_guidance, module)?)?; module.add_function(wrap_pyfunction!(microbatch_guidance, module)?)?; module.add_function(wrap_pyfunction!(explicit_references_help, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs index d01171a01d..db24afda82 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs @@ -1,37 +1,44 @@ -//! Authored locations Python consumers read: YAML entry lines and SQL-test helper spans. +//! Authored locations Python consumers read: YAML entry lines and project-relative paths. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyTuple, PyTupleMethods}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::sql_test_attachment::main::sql_test_cte_location::sql_test_cte_location; -use sqlbuild_analysis::sql_test_attachment::models::CteLocation; use sqlbuild_discovery::yaml_files::main::yaml_entry_lines::yaml_entry_lines; -/// Each outermost `- name:` entry name with its 1-based line. +/// The 1-based line of the outermost `- name: ` entry, if present. #[pyfunction] -fn yaml_entry_line_index(contents: &str) -> Vec<(String, usize)> { +#[pyo3(signature = (*, contents, name))] +fn yaml_entry_line(contents: &str, name: &str) -> Option { yaml_entry_lines(contents) + .into_iter() + .find_map(|(entry, line)| (entry == name).then_some(line)) } -/// `(line, column, end line, end column)` of a helper problem in one SQL test block. +/// `path.relative_to(project_dir)` without pathlib's per-parent comparison scan. #[pyfunction] -#[pyo3(signature = (contents, sql_body, cte_name, call))] -fn sql_test_cte_span( - contents: &str, - sql_body: &str, - cte_name: &str, - call: Option<&str>, -) -> (usize, usize, usize, usize) { - let location: CteLocation = sql_test_cte_location(contents, sql_body, cte_name, call); - ( - location.line, - location.column, - location.end_line, - location.end_column, +#[pyo3(signature = (*, path, project_dir))] +fn project_relative_path<'py>( + path: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let root_parts: Bound<'py, PyTuple> = project_dir.getattr("parts")?.cast_into()?; + let path_parts: Bound<'py, PyTuple> = path.getattr("parts")?.cast_into()?; + let same_anchor: bool = path.call_method0("is_absolute")?.is_truthy()? + == project_dir.call_method0("is_absolute")?.is_truthy()?; + if !same_anchor + || path_parts.len() < root_parts.len() + || !path_parts.get_slice(0, root_parts.len()).eq(&root_parts)? + { + return path.call_method1("relative_to", (project_dir,)); + } + path.call_method1( + "with_segments", + path_parts.get_slice(root_parts.len(), path_parts.len()), ) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(yaml_entry_line_index, module)?)?; - module.add_function(wrap_pyfunction!(sql_test_cte_span, module)?)?; + module.add_function(wrap_pyfunction!(yaml_entry_line, module)?)?; + module.add_function(wrap_pyfunction!(project_relative_path, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs index 3d3638e69d..5173582032 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs @@ -14,6 +14,7 @@ use sqlbuild_discovery::yaml_entries::models::{ use sqlbuild_model_config::errors::{ConfigError, ErrorClass}; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::model_config::contract_classes::yaml_entry_classes; use crate::bindings::_helpers::model_config::header_metadata::{ContractClasses, audit_tuple}; const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; @@ -161,8 +162,8 @@ pub(crate) fn parse_schema_document<'py>( py: Python<'py>, loaded: Bound<'py, PyAny>, path: &str, - classes: YamlClasses<'py>, ) -> PyResult> { + let classes: YamlClasses<'py> = yaml_entry_classes(py)?.extract()?; let in_seeds_directory: bool = std::path::Path::new(path) .components() .any(|component| component.as_os_str() == SEEDS_DIRECTORY); @@ -343,8 +344,8 @@ pub(crate) fn parse_sources_document<'py>( py: Python<'py>, loaded: Bound<'py, PyAny>, path: &str, - classes: YamlClasses<'py>, ) -> PyResult> { + let classes: YamlClasses<'py> = yaml_entry_classes(py)?.extract()?; let sources: Vec>> = parse_sources(&PyNode(loaded), path) .map_err(|error| { parse_error(py, error, SOURCE_PARSE_ERROR).unwrap_or_else(|error| error) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs index 1d33e1f78a..15a8adf641 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs @@ -27,14 +27,13 @@ use crate::bindings::_helpers::discovery::snapshot::{ escaped, in_snapshot, project_root, project_tree, }; use crate::bindings::_helpers::discovery::yaml_entries::{ - YamlClasses, parse_schema_document, parse_sources_document, + parse_schema_document, parse_sources_document, }; use crate::bindings::models::{NativeProjectTree, YamlDiscoveryRequest}; const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; /// The most YAML bytes one native load reads before its files are built and released. const YAML_BATCH_BYTES: u64 = 4 * 1024 * 1024; -const YAML_CLASSES_MODULE: &str = "sqlbuild.compiler.discovery._helpers.yml.yaml_classes"; const PATHLIB_MODULE: &str = "pathlib"; const PROJECT_PATH_KIND: &str = "project_path"; const SCHEMA_KIND: &str = "schema"; @@ -204,12 +203,6 @@ fn unnamed_error(project_dir: &Bound<'_, PyAny>, relative: &Bound<'_, PyAny>) -> )) } -fn yaml_classes(py: Python<'_>) -> PyResult> { - imported(py, YAML_CLASSES_MODULE, "yaml_entry_classes")? - .call0()? - .extract() -} - /// Discover source YAML files natively, reporting failing files to `on_fault`. #[pyfunction] #[pyo3(signature = (*, project_dir, on_fault=None))] @@ -245,8 +238,7 @@ pub(crate) fn discover_source_files<'py>( ("contents", contents.into_bound_py_any(py)?), ( "source_entries", - parse_sources_document(py, loaded, &shown, yaml_classes(py)?)? - .into_any(), + parse_sources_document(py, loaded, &shown)?.into_any(), ), ], ) @@ -307,8 +299,7 @@ pub(crate) fn discover_schema_files<'py>( ("model_entries", PyTuple::empty(py).into_any()), ( "seed_entries", - parse_schema_document(py, loaded, &shown, yaml_classes(py)?)? - .into_any(), + parse_schema_document(py, loaded, &shown)?.into_any(), ), ], )?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs index d4356bf2a5..3d07d2ab76 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::format_command::files::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const FIXES_MODULE: &str = "sqlbuild._native"; -const FIXTURE_AUTOFIX_MODULE: &str = "sqlbuild.compiler.planner.classes.fixture_null_autofix"; +const FIXTURE_AUTOFIX_MODULE: &str = "sqlbuild._native"; const LINT_ERRORS_MODULE: &str = "sqlbuild.lint.exceptions"; const DESCRIPTION_PRESENT: &str = "description-present"; @@ -57,7 +57,7 @@ fn fixture_autofix<'py>( return Ok(PyDict::new(py)); }; Ok(call_with( - &imported(py, FIXTURE_AUTOFIX_MODULE, "FixtureNullAutofix")?.getattr("apply")?, + &imported(py, FIXTURE_AUTOFIX_MODULE, "format_redundant_fixture_nulls")?, &[ ("files", files.clone().into_any()), ("project_dir", request.project_dir.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/exception_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/exception_types.rs new file mode 100644 index 0000000000..8d6c959c03 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/exception_types.rs @@ -0,0 +1,30 @@ +//! Direct source freshness state and observation exceptions. + +use pyo3::create_exception; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +create_exception!( + sqlbuild._native, + SourceFreshnessInputError, + PyValueError, + "Raised when persisted source freshness state cannot be read safely." +); +create_exception!( + sqlbuild._native, + SourceFreshnessObservationError, + PyValueError, + "Raised when source freshness cannot be observed safely." +); + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py = module.py(); + module.add( + "SourceFreshnessInputError", + py.get_type::(), + )?; + module.add( + "SourceFreshnessObservationError", + py.get_type::(), + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index a71149e344..bf3216e726 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -15,9 +15,9 @@ use sqlbuild_core::freshness_text::main::render_freshness_text::render_freshness use sqlbuild_core::freshness_text::models::FreshnessSourceResult; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; -use crate::bindings::_helpers::freshness::freshness_observe::{ - Observer, PreviousState, observe_freshness, -}; +use crate::bindings::_helpers::freshness::freshness_observe::{PreviousState, observe_freshness}; +use crate::bindings::_helpers::freshness::observation::Observer; +use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -29,7 +29,6 @@ const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const UPSTREAM_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; -const STATE_READ_MODULE: &str = "sqlbuild.compiler.source_freshness.main.read"; const STATE_TABLE_NAME: &str = "_sqlbuild_source_freshness"; const STALE_REQUIRES_STATE_MESSAGE: &str = "freshness --fail-on-stale requires --state"; const STALE_REQUIRES_STATE_CODE: &str = "C238"; @@ -180,35 +179,23 @@ fn previous_records<'py>( ], )?), )?; - let read: Bound<'py, PyAny> = imported(py, STATE_READ_MODULE, "read_latest_source_freshness")?; + let read: StateRead<'py> = StateRead { + connection: connection.clone(), + execute: adapter.getattr("execute")?, + render_qualified_name: adapter.getattr("render_qualified_name")?, + render_read_latest_sql: adapter.getattr("render_read_latest_source_freshness_sql")?, + }; let records: Bound<'py, PyDict> = PyDict::new(py); for schema in &schemas { let exists: Bound<'py, PyDict> = keywords(py, &[("database", &database)])?; exists.set_item("schema", schema)?; exists.set_item("name", STATE_TABLE_NAME)?; - let options: Bound<'py, PyDict> = keywords( - py, - &[ - ("connection", connection), - ("execute", &adapter.getattr("execute")?), - ( - "table_exists", - &lookup.call_method("exists", (), Some(&exists))?, - ), - ("database", &database), - ( - "render_qualified_name", - &adapter.getattr("render_qualified_name")?, - ), - ( - "render_read_latest_sql", - &adapter.getattr("render_read_latest_source_freshness_sql")?, - ), - ], - )?; - options.set_item("schema", schema)?; + let table_exists: bool = lookup + .call_method("exists", (), Some(&exists))? + .is_truthy()?; + let schema: Bound<'py, PyAny> = PyString::new(py, schema).into_any(); records.update( - read.call((), Some(&options))? + read_latest(&read, (&database, &schema), table_exists, None)? .getattr("records")? .downcast()?, )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs index 99f698dc69..d98cb58a74 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs @@ -6,23 +6,21 @@ use pyo3::exceptions::{PyException, PyValueError}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyDict, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::freshness_text::main::freshness_json::freshness_json; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::freshness::observation::{ + Observer, observation_record, observation_source, observe_adapter_batch, observe_configured, + strategy, +}; +use crate::bindings::_helpers::state_rows::freshness::{age_policy, equivalent}; use sqlbuild_core::freshness_text::main::freshness_status::freshness_status; use sqlbuild_core::freshness_text::models::{ FreshnessSourceResult, FreshnessStatus, StateComparison, }; -const ADAPTER_OBSERVATION_MODULE: &str = - "sqlbuild.compiler.source_freshness.main.adapter_observation"; -const OBSERVATION_MODULE: &str = "sqlbuild.compiler.source_freshness.main.observation"; -const NATIVE_MODULE: &str = "sqlbuild._native"; -const FRESHNESS_MODELS: &str = "sqlbuild.compiler.source_freshness.models"; -const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; -const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; const RUN_SCOPE_MODULE: &str = "sqlbuild.runtime.observability.main.run_scope"; const ADAPTER_STRATEGY: &str = "adapter"; const UNAVAILABLE_REASON: &str = "unavailable"; @@ -31,25 +29,11 @@ const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavai const MISSING_STATE_MESSAGE: &str = "previous source freshness state missing"; const NOT_RETURNED_MESSAGE: &str = "freshness metadata was not returned"; -/// The adapter, connection and observation time every source is observed with. -pub(crate) struct Observer<'py> { - pub(crate) adapter: Bound<'py, PyAny>, - pub(crate) connection: Bound<'py, PyAny>, - pub(crate) observed_at: Bound<'py, PyAny>, -} - /// Previous state records by identity; absent means no comparison. pub(crate) struct PreviousState<'py> { pub(crate) by_identity: Option>, } -fn freshness_function<'py>( - py: Python<'py>, - (module, name): (&str, &str), -) -> PyResult> { - py.import(module)?.getattr(name) -} - fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { if value.is_none() { return Ok(None); @@ -78,96 +62,6 @@ fn target_result( Ok(result) } -fn strategy(source: &Bound<'_, PyAny>) -> PyResult> { - let freshness: Bound<'_, PyAny> = source.getattr("freshness")?; - if freshness.is_none() { - return Ok(None); - } - optional_text(freshness.getattr("strategy")?) -} - -/// The source as observed: itself with freshness config, an adapter-metadata copy, or none. -fn observation_source<'py>( - adapter: &Bound<'py, PyAny>, - source: &Bound<'py, PyAny>, -) -> PyResult>> { - let py: Python<'py> = source.py(); - if !source.getattr("freshness")?.is_none() { - return Ok(Some(source.clone())); - } - if !source.getattr("expression")?.is_none() - || source.getattr("table")?.is_none() - || !adapter - .call_method0("supports_table_freshness_metadata")? - .is_truthy()? - { - return Ok(None); - } - let config: Bound<'py, PyDict> = PyDict::new(py); - config.set_item( - "strategy", - py.import(CONTRACT_TYPES)? - .getattr("SourceFreshnessStrategy")? - .getattr("ADAPTER")?, - )?; - let changes: Bound<'py, PyDict> = PyDict::new(py); - changes.set_item( - "freshness", - py.import(CONTRACT_MODELS)? - .getattr("SourceFreshnessConfig")? - .call((), Some(&config))?, - )?; - Ok(Some( - py.import("dataclasses")? - .getattr("replace")? - .call((source,), Some(&changes))?, - )) -} - -/// The state record of one observation, with its normalized version and hash. -fn record<'py>( - observation: &Bound<'py, PyAny>, - source: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = observation.py(); - let normalize: Bound<'py, PyDict> = PyDict::new(py); - normalize.set_item("value", observation.getattr("data_version")?)?; - normalize.set_item("value_kind", observation.getattr("value_kind")?)?; - let data_version: Bound<'py, PyAny> = freshness_function( - py, - (NATIVE_MODULE, "normalize_source_freshness_data_version"), - )? - .call((), Some(&normalize))?; - let hash: Bound<'py, PyDict> = PyDict::new(py); - hash.set_item("source_name", observation.getattr("source_name")?)?; - hash.set_item("strategy", observation.getattr("strategy")?)?; - hash.set_item("value_kind", observation.getattr("value_kind")?)?; - hash.set_item("data_version", &data_version)?; - let data_version_hash: Bound<'py, PyAny> = - freshness_function(py, (NATIVE_MODULE, "source_freshness_data_version_hash"))? - .call((), Some(&hash))?; - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item("source_name", observation.getattr("source_name")?)?; - fields.set_item("target_database", source.getattr("database")?)?; - fields.set_item("target_schema", source.getattr("schema")?)?; - fields.set_item("target_name", source.getattr("table")?)?; - fields.set_item("run_id", OBSERVATION_RUN_ID)?; - fields.set_item( - "strategy", - observation.getattr("strategy")?.getattr("value")?, - )?; - fields.set_item( - "value_kind", - observation.getattr("value_kind")?.getattr("value")?, - )?; - fields.set_item("data_version", &data_version)?; - fields.set_item("data_version_hash", data_version_hash)?; - fields.set_item("observed_at", observation.getattr("observed_at")?)?; - py.import(FRESHNESS_MODELS)? - .getattr("SourceFreshnessRecord")? - .call((), Some(&fields)) -} - fn previous_record<'py>( state: &PreviousState<'py>, record: &Bound<'py, PyAny>, @@ -185,7 +79,11 @@ fn observed_result( state: &PreviousState<'_>, ) -> PyResult { let py: Python<'_> = source.py(); - let current: Bound<'_, PyAny> = record(observation, source)?; + let current: Bound<'_, PyAny> = observation_record( + observation, + source, + &PyString::new(py, OBSERVATION_RUN_ID).into_any(), + )?; let freshness: Bound<'_, PyAny> = source.getattr("freshness")?; let (lag_tolerance, policy) = if freshness.is_none() { (py.None().into_bound(py), py.None().into_bound(py)) @@ -195,27 +93,18 @@ fn observed_result( freshness.getattr("age_policy")?, ) }; - let age: Bound<'_, PyDict> = PyDict::new(py); - age.set_item("policy", policy)?; - age.set_item("data_version", observation.getattr("data_version")?)?; - age.set_item("observed_at", observation.getattr("observed_at")?)?; - let age_status: Bound<'_, PyAny> = - freshness_function(py, (NATIVE_MODULE, "evaluate_source_freshness_age_policy"))? - .call((), Some(&age))?; + let age_status: Option> = age_policy( + &policy, + &observation.getattr("data_version")?, + &observation.getattr("observed_at")?, + )?; let previous: Option> = previous_record(state, ¤t)?; let comparison: StateComparison = match &previous { None if state.by_identity.is_some() => StateComparison::Missing, None => StateComparison::NotCompared, Some(previous) => { - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("previous_record", previous)?; - options.set_item("current_record", ¤t)?; - options.set_item("lag_tolerance", &lag_tolerance)?; - let equivalent: bool = - freshness_function(py, (NATIVE_MODULE, "source_freshness_records_equivalent"))? - .call((), Some(&options))? - .is_truthy()?; - if equivalent { + let tolerance: Option = lag_tolerance.extract()?; + if equivalent(previous, ¤t, tolerance.as_deref())? { StateComparison::Equivalent { same_hash: previous .getattr("data_version_hash")? @@ -242,10 +131,9 @@ fn observed_result( if comparison == StateComparison::Missing { result.message = Some(MISSING_STATE_MESSAGE.to_owned()); } - result.age_status = if age_status.is_none() { - None - } else { - Some(age_status.getattr("value")?.extract()?) + result.age_status = match age_status { + None => None, + Some(status) => Some(status.getattr("value")?.extract()?), }; Ok(result) } @@ -256,26 +144,7 @@ fn adapter_batch<'py>( sources: &[Bound<'py, PyAny>], ) -> PyResult, String>> { let py: Python<'py> = observer.adapter.py(); - let batch_class: Bound<'py, PyAny> = py - .import(FRESHNESS_MODELS)? - .getattr("AdapterSourceFreshnessBatch")?; - if sources.is_empty() { - return Ok(Ok(batch_class.call0()?)); - } - let options: Bound<'py, PyDict> = PyDict::new(py); - options.set_item("adapter", &observer.adapter)?; - options.set_item("connection", &observer.connection)?; - options.set_item("sources", PyTuple::new(py, sources)?)?; - options.set_item("observed_at", &observer.observed_at)?; - match freshness_function( - py, - ( - ADAPTER_OBSERVATION_MODULE, - "observe_adapter_sources_freshness", - ), - )? - .call((), Some(&options)) - { + match observe_adapter_batch(observer, sources) { Ok(batch) => Ok(Ok(batch)), Err(error) => Ok(Err(exception_message(py, error)?)), } @@ -329,17 +198,7 @@ fn configured_observation<'py>( (name, source): (&str, &Bound<'py, PyAny>), ) -> PyResult, FreshnessSourceResult>> { let py: Python<'py> = source.py(); - let options: Bound<'py, PyDict> = PyDict::new(py); - options.set_item("adapter", &observer.adapter)?; - options.set_item("connection", &observer.connection)?; - options.set_item("source", source)?; - options.set_item("observed_at", &observer.observed_at)?; - match freshness_function( - py, - (OBSERVATION_MODULE, "observe_configured_source_freshness"), - )? - .call((), Some(&options)) - { + match observe_configured(observer, source) { Ok(observation) => Ok(Ok(observation)), Err(error) => Ok(Err(target_result( source, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs index a2e1f90b98..8d6c6cb4b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs @@ -1,3 +1,8 @@ +pub(crate) mod exception_types; pub(crate) mod freshness_command; pub(crate) mod freshness_observe; +pub(crate) mod observation; +pub(crate) mod planning; pub(crate) mod registration; +pub(crate) mod state_io; +pub(crate) mod support; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs new file mode 100644 index 0000000000..2d8525fd51 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs @@ -0,0 +1,617 @@ +//! Observe configured and adapter-metadata source freshness through adapter callbacks. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_state::freshness::main::single_value_problem::single_value_problem; +use sqlbuild_state::freshness::models::QueryShape; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::freshness::support::{ + ADAPTER_MODELS, ADAPTER_TYPES, CONTRACT_MODELS, CONTRACT_TYPES, FRESHNESS_MODELS, + FRESHNESS_TYPES, LIFECYCLE_MODULE, OBSERVABILITY_MODELS, OBSERVATION_ERROR, exception_text, + freshness_error, presumed_utc_datetime, presumed_utc_value, text, +}; +use crate::bindings::_helpers::references::source_relations::render_source_relation; +use crate::bindings::_helpers::state_rows::freshness::{normalize, version_hash}; +use crate::bindings::_helpers::state_rows::timestamps::{in_utc, offset_aware}; + +const ADAPTER_STRATEGY: &str = "adapter"; +const COLUMN_STRATEGY: &str = "column"; +const DEFAULT_STRATEGY: &str = "sql"; +const METADATA_OPERATION: &str = "source_freshness_metadata_observation"; +const QUERY_OPERATION: &str = "source_freshness_query_observation"; + +/// The adapter, connection and observation time every source is observed with. +pub(crate) struct Observer<'py> { + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) connection: Bound<'py, PyAny>, + pub(crate) observed_at: Bound<'py, PyAny>, +} + +fn observation_error(py: Python<'_>, message: &str) -> pyo3::PyErr { + freshness_error(py, (OBSERVATION_ERROR, message), None) +} + +/// The configured strategy value of a source, or `None` without freshness config. +pub(crate) fn strategy(source: &Bound<'_, PyAny>) -> PyResult> { + let freshness: Bound<'_, PyAny> = source.getattr("freshness")?; + if freshness.is_none() { + return Ok(None); + } + Ok(Some(freshness.getattr("strategy")?.str()?.extract()?)) +} + +/// The source as observed: itself with freshness config, an adapter-metadata copy, or none. +pub(crate) fn observation_source<'py>( + adapter: &Bound<'py, PyAny>, + source: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = source.py(); + if !source.getattr("freshness")?.is_none() { + return Ok(Some(source.clone())); + } + if !source.getattr("expression")?.is_none() + || source.getattr("table")?.is_none() + || !adapter + .call_method0("supports_table_freshness_metadata")? + .is_truthy()? + { + return Ok(None); + } + let config: Bound<'py, PyAny> = call_with( + &imported(py, CONTRACT_MODELS, "SourceFreshnessConfig")?, + &[("strategy", adapter_strategy(py)?)], + )?; + replaced_source(source, config).map(Some) +} + +fn replaced_source<'py>( + source: &Bound<'py, PyAny>, + config: Bound<'py, PyAny>, +) -> PyResult> { + let changes: Bound<'py, PyDict> = PyDict::new(source.py()); + changes.set_item("freshness", config)?; + imported(source.py(), "dataclasses", "replace")?.call((source,), Some(&changes)) +} + +fn adapter_strategy(py: Python<'_>) -> PyResult> { + imported(py, CONTRACT_TYPES, "SourceFreshnessStrategy")?.getattr("ADAPTER") +} + +/// `OperationLifecycle(operation_kind="freshness", ...)` for one observation phase. +fn lifecycle<'py>( + py: Python<'py>, + (operation_name, strategy): (&str, &str), + item_count: Option, +) -> PyResult> { + let attributes: Bound<'py, PyAny> = call_with( + &imported(py, OBSERVABILITY_MODELS, "OperationAttributes")?, + &[ + ("phase", text(py, "observe")), + ("strategy", text(py, strategy)), + ("target_kind", text(py, "source")), + ], + )?; + let mut options: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("operation_kind", text(py, "freshness")), + ("operation_name", text(py, operation_name)), + ]; + if let Some(item_count) = item_count { + let metadata: Bound<'py, PyDict> = PyDict::new(py); + metadata.set_item("item_count", item_count)?; + options.push(("metadata", metadata.into_any())); + } + options.push(("attributes", attributes)); + call_with( + &imported(py, LIFECYCLE_MODULE, "OperationLifecycle")?, + &options, + ) +} + +fn observation<'py>( + source_name: &Bound<'py, PyAny>, + (strategy, value_kind): (Bound<'py, PyAny>, Bound<'py, PyAny>), + (data_version, observed_at): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + call_with( + &imported( + source_name.py(), + FRESHNESS_MODELS, + "SourceFreshnessObservation", + )?, + &[ + ("source_name", source_name.clone()), + ("strategy", strategy), + ("data_version", data_version), + ("value_kind", value_kind), + ("observed_at", observed_at), + ], + ) +} + +/// Observe one source freshness config and return a comparable data version. +pub(crate) fn observe_configured<'py>( + observer: &Observer<'py>, + source: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = source.py(); + let strategy: Option = strategy(source)?; + let operation: &str = if strategy.as_deref() == Some(ADAPTER_STRATEGY) { + METADATA_OPERATION + } else { + QUERY_OPERATION + }; + let manager: Bound<'py, PyAny> = lifecycle( + py, + (operation, strategy.as_deref().unwrap_or(DEFAULT_STRATEGY)), + None, + )?; + within(&manager, |_| { + observe_configured_inner(observer, source, strategy.as_deref()) + }) +} + +fn observe_configured_inner<'py>( + observer: &Observer<'py>, + source: &Bound<'py, PyAny>, + strategy: Option<&str>, +) -> PyResult> { + let py: Python<'py> = source.py(); + let name: Bound<'py, PyAny> = source.getattr("name")?; + let config: Bound<'py, PyAny> = source.getattr("freshness")?; + let Some(strategy) = strategy else { + return Err(observation_error( + py, + &format!( + "source '{}' does not define freshness configuration", + name.str()? + ), + )); + }; + if strategy == ADAPTER_STRATEGY { + return observe_adapter_source(observer, source); + } + let value_kind: Bound<'py, PyAny> = config.getattr("value_kind")?; + let sql: Bound<'py, PyAny> = if strategy == COLUMN_STRATEGY { + let column: Bound<'py, PyAny> = config.getattr("column")?; + if column.is_none() || value_kind.is_none() { + return Err(observation_error( + py, + &format!( + "source '{}' has incomplete column freshness configuration", + name.str()? + ), + )); + } + let filter: Bound<'py, PyAny> = config.getattr("filter")?; + let where_sql: String = if filter.is_none() { + String::new() + } else { + format!(" WHERE {}", filter.str()?) + }; + let relation: String = render_source_relation(source, Some(&observer.adapter))?; + call_with( + &observer + .adapter + .getattr("render_source_freshness_max_query")?, + &[ + ("column", column), + ("source_relation", text(py, &relation)), + ( + "source_is_subquery", + PyBool::new(py, !source.getattr("expression")?.is_none()) + .to_owned() + .into_any(), + ), + ("where_sql", text(py, &where_sql)), + ], + )? + } else { + let query: Bound<'py, PyAny> = config.getattr("query")?; + if query.is_none() || value_kind.is_none() { + return Err(observation_error( + py, + &format!( + "source '{}' has incomplete sql freshness configuration", + name.str()? + ), + )); + } + query + }; + let data_version: Bound<'py, PyAny> = + presumed_utc_value(&single_data_version(observer, &name, sql)?)?; + observation( + &name, + (config.getattr("strategy")?, value_kind), + (data_version, observer.observed_at.clone()), + ) +} + +fn single_data_version<'py>( + observer: &Observer<'py>, + source_name: &Bound<'py, PyAny>, + sql: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = source_name.py(); + let name: String = source_name.str()?.extract()?; + let result: Bound<'py, PyAny> = match call_with( + &observer.adapter.getattr("query")?, + &[ + ("connection", observer.connection.clone()), + ("sql", sql), + ("limit", py.None().into_bound(py)), + ], + ) { + Ok(result) => result, + Err(error) => { + let Some(message) = exception_text(py, &error)? else { + return Err(error); + }; + return Err(freshness_error( + py, + ( + OBSERVATION_ERROR, + &format!("source '{name}' freshness query failed: {message}"), + ), + Some(error), + )); + } + }; + let columns: usize = result.getattr("columns")?.len()?; + let rows: Bound<'py, PyAny> = result.getattr("rows")?; + let row_count: usize = rows.len()?; + let first: Option> = if columns == 1 && row_count == 1 { + Some(rows.get_item(0)?.get_item(0)?) + } else { + None + }; + let shape: QueryShape = QueryShape { + columns, + rows: row_count, + first_value_null: first.as_ref().is_some_and(|value| value.is_none()), + }; + if let Some(problem) = single_value_problem(&name, shape) { + return Err(observation_error(py, &problem)); + } + first.ok_or_else(|| observation_error(py, "freshness query returned no value")) +} + +fn unsupported_message( + adapter: &Bound<'_, PyAny>, + source: &Bound<'_, PyAny>, +) -> PyResult> { + if !adapter + .call_method0("supports_table_freshness_metadata")? + .is_truthy()? + { + return Ok(Some(format!( + "adapter '{}' does not support table freshness metadata", + adapter.getattr("adapter_name")?.str()? + ))); + } + if !source.getattr("expression")?.is_none() || source.getattr("table")?.is_none() { + return Ok(Some(format!( + "source '{}' adapter freshness requires a physical table source", + source.getattr("name")?.str()? + ))); + } + Ok(None) +} + +/// `SourceFreshnessUnknown(source_name=..., reason=, message=...)`. +pub(crate) fn unknown<'py>( + source_name: &Bound<'py, PyAny>, + (reason, message): (&str, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = source_name.py(); + call_with( + &imported(py, FRESHNESS_MODELS, "SourceFreshnessUnknown")?, + &[ + ("source_name", source_name.clone()), + ( + "reason", + imported(py, FRESHNESS_TYPES, "SourceFreshnessUnknownReason")?.getattr(reason)?, + ), + ("message", message), + ], + ) +} + +/// The unknown outcome of unavailable metadata or a null data version, if any. +fn metadata_unknown<'py>( + source_name: &Bound<'py, PyAny>, + metadata: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = source_name.py(); + let name: String = source_name.str()?.extract()?; + let unavailable: Bound<'py, PyAny> = + imported(py, ADAPTER_TYPES, "TableFreshnessStatus")?.getattr("UNAVAILABLE")?; + if metadata.getattr("status")?.eq(unavailable)? { + let message: Bound<'py, PyAny> = metadata.getattr("message")?; + let message: String = if message.is_truthy()? { + message.str()?.extract()? + } else { + format!("source '{name}' freshness metadata unavailable") + }; + return unknown(source_name, ("UNAVAILABLE", text(py, &message))).map(Some); + } + if metadata.getattr("data_version")?.is_none() { + let message: String = format!("source '{name}' freshness data_version is null"); + return unknown(source_name, ("UNAVAILABLE", text(py, &message))).map(Some); + } + Ok(None) +} + +fn metadata_observation<'py>( + source_name: &Bound<'py, PyAny>, + metadata: &Bound<'py, PyAny>, + fallback_observed_at: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = source_name.py(); + let observed_at: Bound<'py, PyAny> = metadata.getattr("observed_at")?; + let observed_at: Bound<'py, PyAny> = if observed_at.is_none() { + fallback_observed_at.clone() + } else { + presumed_utc_datetime(&observed_at)? + }; + observation( + source_name, + ( + adapter_strategy(py)?, + imported(py, CONTRACT_TYPES, "SourceFreshnessValueKind")? + .call1((metadata.getattr("value_kind")?,))?, + ), + ( + presumed_utc_value(&metadata.getattr("data_version")?)?, + observed_at, + ), + ) +} + +fn observe_adapter_source<'py>( + observer: &Observer<'py>, + source: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = source.py(); + if let Some(message) = unsupported_message(&observer.adapter, source)? { + return Err(observation_error(py, &message)); + } + let metadata: Bound<'py, PyAny> = call_with( + &observer.adapter.getattr("get_table_freshness_metadata")?, + &[ + ("connection", observer.connection.clone()), + ("database", source.getattr("database")?), + ("schema", source.getattr("schema")?), + ("name", source.getattr("table")?), + ], + )?; + let name: Bound<'py, PyAny> = source.getattr("name")?; + if let Some(unknown) = metadata_unknown(&name, &metadata)? { + return Err(observation_error( + py, + &unknown.getattr("message")?.str()?.extract::()?, + )); + } + metadata_observation(&name, &metadata, &observer.observed_at) +} + +/// Observe adapter metadata freshness per source, batching the warehouse lookups. +pub(crate) fn observe_adapter_batch<'py>( + observer: &Observer<'py>, + sources: &[Bound<'py, PyAny>], +) -> PyResult> { + let py: Python<'py> = observer.adapter.py(); + let batch_class: Bound<'py, PyAny> = + imported(py, FRESHNESS_MODELS, "AdapterSourceFreshnessBatch")?; + if sources.is_empty() { + return batch_class.call0(); + } + let manager: Bound<'py, PyAny> = lifecycle( + py, + (METADATA_OPERATION, ADAPTER_STRATEGY), + Some(sources.len()), + )?; + within(&manager, |entered| { + let batch: Bound<'py, PyAny> = observe_adapter_batch_inner(observer, sources)?; + let metadata: Bound<'py, PyDict> = PyDict::new(py); + metadata.set_item("item_count", sources.len())?; + metadata.set_item("unknown_count", batch.getattr("unknown")?.len()?)?; + call_with( + &entered.getattr("completed")?, + &[("metadata", metadata.into_any())], + )?; + Ok(batch) + }) +} + +fn observe_adapter_batch_inner<'py>( + observer: &Observer<'py>, + sources: &[Bound<'py, PyAny>], +) -> PyResult> { + let py: Python<'py> = observer.adapter.py(); + let batch_class: Bound<'py, PyAny> = + imported(py, FRESHNESS_MODELS, "AdapterSourceFreshnessBatch")?; + let unknown_sources: Bound<'py, PyDict> = PyDict::new(py); + let requests: Bound<'py, PyDict> = PyDict::new(py); + let request_class: Bound<'py, PyAny> = imported(py, ADAPTER_MODELS, "TableFreshnessRequest")?; + for source in sources { + let name: Bound<'py, PyAny> = source.getattr("name")?; + if let Some(message) = unsupported_message(&observer.adapter, source)? { + unknown_sources + .set_item(&name, unknown(&name, ("UNAVAILABLE", text(py, &message)))?)?; + continue; + } + requests.set_item( + &name, + call_with( + &request_class, + &[ + ("database", source.getattr("database")?), + ("schema", source.getattr("schema")?), + ("name", source.getattr("table")?.str()?.into_any()), + ], + )?, + )?; + } + if requests.is_empty() { + return call_with(&batch_class, &[("unknown", unknown_sources.into_any())]); + } + let unique: Bound<'py, PyDict> = PyDict::new(py); + for request in requests.values() { + unique.set_item(request, py.None())?; + } + let metadata_by_request: Bound<'py, PyAny> = call_with( + &observer.adapter.getattr("get_tables_freshness_metadata")?, + &[ + ("connection", observer.connection.clone()), + ("requests", PyTuple::new(py, unique.keys())?.into_any()), + ], + )?; + let metadata_class: Bound<'py, PyAny> = imported(py, ADAPTER_MODELS, "TableFreshnessMetadata")?; + let observations: Bound<'py, PyDict> = PyDict::new(py); + for (name, request) in requests.iter() { + let missing: Bound<'py, PyAny> = call_with( + &metadata_class.getattr("unavailable")?, + &[( + "message", + text( + py, + &format!( + "source '{}' freshness metadata was not returned", + name.str()? + ), + ), + )], + )?; + let metadata: Bound<'py, PyAny> = + metadata_by_request.call_method1("get", (request, missing))?; + if let Some(unknown) = metadata_unknown(&name, &metadata)? { + unknown_sources.set_item(&name, unknown)?; + continue; + } + observations.set_item( + &name, + metadata_observation(&name, &metadata, &observer.observed_at)?, + )?; + } + call_with( + &batch_class, + &[ + ("observations", observations.into_any()), + ("unknown", unknown_sources.into_any()), + ], + ) +} + +/// The state record of one observation, with its normalized version and hash. +pub(crate) fn observation_record<'py>( + observation: &Bound<'py, PyAny>, + source: &Bound<'py, PyAny>, + run_id: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = observation.py(); + let source_name: Bound<'py, PyAny> = observation.getattr("source_name")?; + let strategy: Bound<'py, PyAny> = observation.getattr("strategy")?; + let value_kind: Bound<'py, PyAny> = observation.getattr("value_kind")?; + let data_version: Bound<'py, PyAny> = + normalize(&observation.getattr("data_version")?, &value_kind)?; + let data_version_hash: String = version_hash( + &source_name.extract::()?, + &strategy.extract::()?, + &value_kind.extract::()?, + &data_version.extract::()?, + )?; + call_with( + &imported(py, FRESHNESS_MODELS, "SourceFreshnessRecord")?, + &[ + ("source_name", source_name), + ("target_database", source.getattr("database")?), + ("target_schema", source.getattr("schema")?), + ("target_name", source.getattr("table")?), + ("run_id", run_id.clone()), + ("strategy", strategy.getattr("value")?), + ("value_kind", value_kind.getattr("value")?), + ("data_version", data_version), + ("data_version_hash", text(py, &data_version_hash)), + ("observed_at", observation.getattr("observed_at")?), + ], + ) +} + +/// Observe one source freshness config and return a comparable data version. +#[pyfunction] +#[pyo3(signature = (*, adapter, connection, source, observed_at))] +fn observe_configured_source_freshness<'py>( + adapter: Bound<'py, PyAny>, + connection: Bound<'py, PyAny>, + source: &Bound<'py, PyAny>, + observed_at: Bound<'py, PyAny>, +) -> PyResult> { + let observer: Observer<'py> = Observer { + adapter, + connection, + observed_at, + }; + observe_configured(&observer, source) +} + +/// Observe adapter metadata freshness per source, batching the warehouse lookups. +#[pyfunction] +#[pyo3(signature = (*, adapter, connection, sources, observed_at))] +fn observe_adapter_sources_freshness<'py>( + adapter: Bound<'py, PyAny>, + connection: Bound<'py, PyAny>, + sources: &Bound<'py, PyAny>, + observed_at: Bound<'py, PyAny>, +) -> PyResult> { + let observer: Observer<'py> = Observer { + adapter, + connection, + observed_at, + }; + let sources: Vec> = sources.try_iter()?.collect::>()?; + observe_adapter_batch(&observer, &sources) +} + +/// Build the state record of one observation. +#[pyfunction] +#[pyo3(signature = (*, observation, source, run_id))] +fn source_freshness_record_from_observation<'py>( + observation: &Bound<'py, PyAny>, + source: &Bound<'py, PyAny>, + run_id: &Bound<'py, PyAny>, +) -> PyResult> { + observation_record(observation, source, run_id) +} + +/// An aware freshness `datetime` in UTC; a naive value is an observation error. +#[pyfunction] +#[pyo3(signature = (*, value, field_name))] +fn require_aware_utc_datetime<'py>( + value: &Bound<'py, PyAny>, + field_name: &str, +) -> PyResult> { + if !offset_aware(value)? { + return Err(observation_error( + value.py(), + &format!("source freshness {field_name} must be timezone-aware"), + )); + } + in_utc(value) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(require_aware_utc_datetime, module)?)?; + module.add_function(wrap_pyfunction!( + observe_configured_source_freshness, + module + )?)?; + module.add_function(wrap_pyfunction!(observe_adapter_sources_freshness, module)?)?; + module.add_function(wrap_pyfunction!( + source_freshness_record_from_observation, + module + )?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs new file mode 100644 index 0000000000..e352a88857 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs @@ -0,0 +1,388 @@ +//! Planning-time source freshness: observe, compare with latest state and map downstream impact. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_state::freshness::main::configuration_error::configuration_error; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::freshness::observation::{ + Observer, observation_record, observation_source, observe_adapter_batch, observe_configured, + strategy, unknown, +}; +use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; +use crate::bindings::_helpers::freshness::support::{ + ADAPTER_EXCEPTIONS, EXCEPTIONS_MODULE, FRESHNESS_MODELS, FRESHNESS_TYPES, OBSERVATION_ERROR, + text, +}; +use crate::bindings::_helpers::planner_identity::graph::build_downstream_model_name_closure; +use crate::bindings::_helpers::state_rows::freshness::{age_policy, equivalent}; + +const ADAPTER_STRATEGY: &str = "adapter"; +const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; +const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavailable"; + +/// The message of an `AdapterUserError` or a source freshness observation error, if it is one. +fn handled_message<'py>(py: Python<'py>, error: &PyErr) -> PyResult>> { + let value: Bound<'py, PyAny> = error.value(py).clone().into_any(); + if value.is_instance(&imported(py, ADAPTER_EXCEPTIONS, "AdapterUserError")?)? { + return value.getattr("message").map(Some); + } + if value.is_instance(&imported(py, EXCEPTIONS_MODULE, OBSERVATION_ERROR)?)? { + return Ok(Some(value.str()?.into_any())); + } + Ok(None) +} + +fn freshness_strategy_is_adapter(source: &Bound<'_, PyAny>) -> PyResult { + Ok(strategy(source)?.as_deref() == Some(ADAPTER_STRATEGY)) +} + +/// Latest previous records by identity across the state schemas, newest observation winning. +fn previous_records<'py>( + arguments: &KeywordArguments<'py>, + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + source_names: Option<&[String]>, +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let read: StateRead<'py> = StateRead { + connection: connection.clone(), + execute: adapter.getattr("execute")?, + render_qualified_name: arguments.value("render_qualified_name")?, + render_read_latest_sql: adapter.getattr("render_read_latest_source_freshness_sql")?, + }; + let database: Bound<'py, PyAny> = arguments.value("state_database")?; + let exists_by_schema: Bound<'py, PyAny> = arguments.value("state_table_exists_by_schema")?; + let merged: Bound<'py, PyDict> = PyDict::new(py); + for schema in arguments.value("state_schemas")?.try_iter()? { + let schema: Bound<'py, PyAny> = schema?; + let exists: bool = exists_by_schema + .call_method1("get", (&schema, false))? + .is_truthy()?; + let previous: Bound<'py, PyAny> = + read_latest(&read, (&database, &schema), exists, source_names)?; + for (identity, candidate) in previous.getattr("records")?.downcast::()?.iter() { + let replace: bool = match merged.get_item(&identity)? { + None => true, + Some(current) => candidate + .getattr("observed_at")? + .gt(current.getattr("observed_at")?)?, + }; + if replace { + merged.set_item(identity, candidate)?; + } + } + } + Ok(merged) +} + +/// Records sorted by `str(record.identity)`, stably. +fn sorted_by_identity<'py>( + records: impl Iterator>, +) -> PyResult>> { + let mut keyed: Vec<(String, Bound<'py, PyAny>)> = records + .map(|record| Ok((record.getattr("identity")?.str()?.extract()?, record))) + .collect::>()?; + keyed.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(keyed.into_iter().map(|(_, record)| record).collect()) +} + +fn sorted_dict<'py>(py: Python<'py>, items: &Bound<'py, PyDict>) -> PyResult> { + let mut keyed: Vec<(String, Bound<'py, PyAny>)> = items + .iter() + .map(|(name, value)| Ok((name.extract()?, value))) + .collect::>()?; + keyed.sort_by(|left, right| left.0.cmp(&right.0)); + let sorted: Bound<'py, PyDict> = PyDict::new(py); + for (name, value) in keyed { + sorted.set_item(name, value)?; + } + Ok(sorted) +} + +/// The adapter batch, or every source unknown with the batch failure's message. +fn adapter_batch<'py>( + observer: &Observer<'py>, + sources: &[Bound<'py, PyAny>], +) -> PyResult> { + let py: Python<'py> = observer.adapter.py(); + let error: PyErr = match observe_adapter_batch(observer, sources) { + Ok(batch) => return Ok(batch), + Err(error) => error, + }; + let Some(message) = handled_message(py, &error)? else { + return Err(error); + }; + let unknown_sources: Bound<'py, PyDict> = PyDict::new(py); + for source in sources { + let name: Bound<'py, PyAny> = source.getattr("name")?; + unknown_sources.set_item(&name, unknown(&name, ("ERROR", message.clone()))?)?; + } + call_with( + &imported(py, FRESHNESS_MODELS, "AdapterSourceFreshnessBatch")?, + &[("unknown", unknown_sources.into_any())], + ) +} + +/// Observe direct source freshness and compare it to latest append-only state. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_direct_source_freshness_planning_result<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_direct_source_freshness_planning_result", + options, + ( + &[ + "adapter", + "connection", + "sources", + "state_database", + "state_schemas", + "observed_at", + "run_id", + "render_qualified_name", + "state_table_exists_by_schema", + ], + &["filter_previous_to_sources"], + ), + )?; + let observer: Observer<'py> = Observer { + adapter: arguments.value("adapter")?, + connection: arguments.value("connection")?, + observed_at: arguments.value("observed_at")?, + }; + let run_id: Bound<'py, PyAny> = arguments.value("run_id")?; + let sources: Vec> = arguments + .value("sources")? + .try_iter()? + .collect::>()?; + let source_names: Option> = + if arguments.value("filter_previous_to_sources")?.is_truthy()? { + let mut names: Vec = sources + .iter() + .map(|source| source.getattr("name")?.extract()) + .collect::>()?; + names.sort(); + Some(names) + } else { + None + }; + let previous: Bound<'py, PyDict> = previous_records( + &arguments, + (&observer.adapter, &observer.connection), + source_names.as_deref(), + )?; + let unknown_sources: Bound<'py, PyDict> = PyDict::new(py); + let observed_sources: Bound<'py, PyDict> = PyDict::new(py); + let mut adapter_sources: Vec> = Vec::new(); + for source in &sources { + if source.getattr("managed")?.is_truthy()? { + continue; + } + let name: Bound<'py, PyAny> = source.getattr("name")?; + let Some(observed) = observation_source(&observer.adapter, source)? else { + unknown_sources.set_item( + &name, + unknown(&name, ("NO_FRESHNESS_CONFIG", text(py, NO_CONFIG_MESSAGE)))?, + )?; + continue; + }; + observed_sources.set_item(&name, &observed)?; + if freshness_strategy_is_adapter(&observed)? { + adapter_sources.push(observed); + } + } + let batch: Bound<'py, PyAny> = adapter_batch(&observer, &adapter_sources)?; + unknown_sources.update(batch.getattr("unknown")?.downcast()?)?; + let observations: Bound<'py, PyAny> = batch.getattr("observations")?; + let mut observed_records: Vec> = Vec::new(); + let changed: Bound<'py, PySet> = PySet::empty(py)?; + let unchanged: Bound<'py, PySet> = PySet::empty(py)?; + let age_statuses: Bound<'py, PyDict> = PyDict::new(py); + for (name, source) in observed_sources.iter() { + let observation: Bound<'py, PyAny> = if freshness_strategy_is_adapter(&source)? { + let observation: Bound<'py, PyAny> = observations.call_method1("get", (&name,))?; + if observation.is_none() { + continue; + } + observation + } else { + match observe_configured(&observer, &source) { + Ok(observation) => observation, + Err(error) => { + let Some(message) = handled_message(py, &error)? else { + return Err(error); + }; + let is_observation_error: bool = !error.value(py).is_instance(&imported( + py, + ADAPTER_EXCEPTIONS, + "AdapterUserError", + )?)?; + if is_observation_error && configuration_error(&message.extract::()?) { + return Err(error); + } + unknown_sources.set_item(&name, unknown(&name, ("ERROR", message))?)?; + continue; + } + } + }; + let record: Bound<'py, PyAny> = observation_record(&observation, &source, &run_id)?; + observed_records.push(record.clone()); + let freshness: Bound<'py, PyAny> = source.getattr("freshness")?; + let (policy, lag_tolerance) = if freshness.is_none() { + (py.None().into_bound(py), None) + } else { + ( + freshness.getattr("age_policy")?, + freshness + .getattr("lag_tolerance")? + .extract::>()?, + ) + }; + let identity: Bound<'py, PyAny> = record.getattr("identity")?; + if let Some(status) = age_policy( + &policy, + &observation.getattr("data_version")?, + &observation.getattr("observed_at")?, + )? { + age_statuses.set_item(&identity, status)?; + } + match previous.get_item(&identity)? { + None => changed.add(&identity)?, + Some(previous_record) => { + if equivalent(&previous_record, &record, lag_tolerance.as_deref())? { + unchanged.add(&identity)?; + } else { + changed.add(&identity)?; + } + } + } + } + let unknown_names: Bound<'py, PyDict> = sorted_dict(py, &unknown_sources)?; + call_with( + &imported(py, FRESHNESS_MODELS, "DirectSourceFreshnessPlanningResult")?, + &[ + ( + "observed_records", + PyTuple::new(py, sorted_by_identity(observed_records.into_iter())?)?.into_any(), + ), + ( + "previous_records", + PyTuple::new(py, sorted_by_identity(previous.values().into_iter())?)?.into_any(), + ), + ("changed_identities", frozen(&changed)?), + ("unchanged_identities", frozen(&unchanged)?), + ( + "unknown_source_names", + PyTuple::new(py, unknown_names.keys())?.into_any(), + ), + ("unknown_sources", unknown_names.into_any()), + ("age_statuses", age_statuses.into_any()), + ], + ) +} + +/// `frozenset(items)`, keeping CPython's copied set layout and iteration order. +fn frozen<'py>(items: &Bound<'py, PySet>) -> PyResult> { + items.py().get_type::().call1((items,)) +} + +fn downstream_models<'py>( + source_name: &Bound<'py, PyAny>, + downstream_deps: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = source_name.py(); + let key: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS, "CompiledObjectKey")?, + &[ + ( + "resource_type", + imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SOURCE")?, + ), + ("name", source_name.clone()), + ], + )?; + build_downstream_model_name_closure(&PyTuple::new(py, [key])?.into_any(), downstream_deps) +} + +/// Map changed and unknown source freshness roots to downstream model names. +#[pyfunction] +#[pyo3(signature = (*, source_freshness, scope))] +pub(crate) fn build_direct_source_freshness_propagation_result<'py>( + source_freshness: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = scope.py(); + let downstream_deps: Bound<'py, PyAny> = scope.getattr("downstream_deps")?; + let changed: Bound<'py, PyDict> = PyDict::new(py); + let unknown_sources: Bound<'py, PyDict> = PyDict::new(py); + let errors: Bound<'py, PyDict> = PyDict::new(py); + let stale: Bound<'py, PySet> = PySet::empty(py)?; + let blocked: Bound<'py, PySet> = PySet::empty(py)?; + for identity in source_freshness.getattr("changed_identities")?.try_iter()? { + let identity: Bound<'py, PyAny> = identity?; + let models: Bound<'py, PyFrozenSet> = + downstream_models(&identity.getattr("source_name")?, &downstream_deps)?; + stale.call_method1("update", (&models,))?; + changed.set_item(identity, models)?; + } + for name in source_freshness + .getattr("unknown_source_names")? + .try_iter()? + { + let name: Bound<'py, PyAny> = name?; + let models: Bound<'py, PyFrozenSet> = downstream_models(&name, &downstream_deps)?; + stale.call_method1("update", (&models,))?; + unknown_sources.set_item(name, models)?; + } + let error_status: Bound<'py, PyAny> = + imported(py, FRESHNESS_TYPES, "SourceFreshnessAgeStatus")?.getattr("ERROR")?; + for (identity, status) in source_freshness + .getattr("age_statuses")? + .downcast::()? + .iter() + { + if status.ne(&error_status)? { + continue; + } + let models: Bound<'py, PyFrozenSet> = + downstream_models(&identity.getattr("source_name")?, &downstream_deps)?; + blocked.call_method1("update", (&models,))?; + errors.set_item(identity, models)?; + } + call_with( + &imported( + py, + FRESHNESS_MODELS, + "DirectSourceFreshnessPropagationResult", + )?, + &[ + ("changed_source_model_names", changed.into_any()), + ("unknown_source_model_names", unknown_sources.into_any()), + ("error_source_model_names", errors.into_any()), + ("stale_model_names", frozen(&stale)?), + ("blocked_model_names", frozen(&blocked)?), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_direct_source_freshness_planning_result, + module + )?)?; + module.add_function(wrap_pyfunction!( + build_direct_source_freshness_propagation_result, + module + )?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs index 0f2cffdbe0..0d0d44f770 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs @@ -1,10 +1,16 @@ -//! Register the native `sqb freshness` command and its observation entry. +//! Register the native `sqb freshness` command, observation, planning and state functions. use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::freshness::{freshness_command, freshness_observe}; +use crate::bindings::_helpers::freshness::{ + exception_types, freshness_command, freshness_observe, observation, planning, state_io, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + exception_types::register(module)?; freshness_command::register(module)?; - freshness_observe::register(module) + freshness_observe::register(module)?; + observation::register(module)?; + planning::register(module)?; + state_io::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs new file mode 100644 index 0000000000..be7f68cdd4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs @@ -0,0 +1,332 @@ +//! Source freshness state table DDL, latest-row reads and appends through adapter callbacks. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_state::freshness::constants::{REQUIRED_STATE_COLUMNS, SOURCE_FRESHNESS_TABLE_NAME}; +use sqlbuild_state::freshness::main::read_latest_sql::read_latest_sql; +use sqlbuild_state::freshness::main::relevant_sources_sql::relevant_sources_sql; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; +use crate::bindings::_helpers::freshness::support::{ + FRESHNESS_CONSTANTS, FRESHNESS_MODELS, INPUT_ERROR, STATE_TABLE_DDL_MODULE, exception_text, + freshness_error, optional_str, presumed_utc_datetime, text, +}; +use crate::bindings::_helpers::state_rows::timestamps::date_time_typed; + +const MISSING_SCHEMA_MESSAGE: &str = "source freshness table requires a target schema"; + +/// The qualified state table name the adapter renders for one target schema. +pub(crate) fn qualified_table_name( + render_qualified_name: &Bound<'_, PyAny>, + (database, schema): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult { + let py: Python<'_> = render_qualified_name.py(); + let name: Bound<'_, PyAny> = call_with( + render_qualified_name, + &[ + ("database", database.clone()), + ("schema", schema.clone()), + ("name", text(py, SOURCE_FRESHNESS_TABLE_NAME)), + ], + )?; + if name.is_none() { + return Err(freshness_error( + py, + (INPUT_ERROR, MISSING_SCHEMA_MESSAGE), + None, + )); + } + name.extract() +} + +/// SQL selecting the latest source freshness row per identity. +#[pyfunction] +#[pyo3(signature = (*, database, schema, render_qualified_name))] +fn build_source_freshness_read_latest_sql( + database: &Bound<'_, PyAny>, + schema: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, +) -> PyResult { + Ok(read_latest_sql(&qualified_table_name( + render_qualified_name, + (database, schema), + )?)) +} + +/// Create-if-missing DDL for the source freshness state table. +#[pyfunction] +#[pyo3(signature = (*, database, schema, render_qualified_name, render_framework_type))] +fn build_source_freshness_create_table_sql<'py>( + database: &Bound<'py, PyAny>, + schema: &Bound<'py, PyAny>, + render_qualified_name: &Bound<'py, PyAny>, + render_framework_type: &Bound<'py, PyAny>, +) -> PyResult> { + create_table_sql( + (database, schema), + render_qualified_name, + render_framework_type, + ) +} + +fn create_table_sql<'py>( + (database, schema): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + render_qualified_name: &Bound<'py, PyAny>, + render_framework_type: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = database.py(); + let qualified_name: String = qualified_table_name(render_qualified_name, (database, schema))?; + let constants: Bound<'py, PyModule> = py.import(FRESHNESS_CONSTANTS)?; + call_with( + &imported(py, STATE_TABLE_DDL_MODULE, "render_state_table_create_sql")?, + &[ + ("qualified_name", text(py, &qualified_name)), + ("columns", constants.getattr("SOURCE_FRESHNESS_COLUMNS")?), + ( + "column_types", + constants.getattr("SOURCE_FRESHNESS_COLUMN_TYPES")?, + ), + ( + "required_columns", + PyFrozenSet::new(py, REQUIRED_STATE_COLUMNS)?.into_any(), + ), + ("render_framework_type", render_framework_type.clone()), + ], + ) +} + +/// The callbacks and location one latest-state read uses. +pub(crate) struct StateRead<'py> { + pub(crate) connection: Bound<'py, PyAny>, + pub(crate) execute: Bound<'py, PyAny>, + pub(crate) render_qualified_name: Bound<'py, PyAny>, + pub(crate) render_read_latest_sql: Bound<'py, PyAny>, +} + +fn empty_set<'py>(schema: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = schema.py(); + call_with( + &imported(py, FRESHNESS_MODELS, "SourceFreshnessSet")?, + &[ + ("schema", schema.clone()), + ("records", PyDict::new(py).into_any()), + ], + ) +} + +/// Latest records of one schema, trusting the caller-resolved table existence. +pub(crate) fn read_latest<'py>( + read: &StateRead<'py>, + (database, schema): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + table_exists: bool, + source_names: Option<&[String]>, +) -> PyResult> { + let py: Python<'py> = schema.py(); + let qualified_name: String = + qualified_table_name(&read.render_qualified_name, (database, schema))?; + if !table_exists { + return empty_set(schema); + } + let mut read_sql: String = call_with( + &read.render_read_latest_sql, + &[("database", database.clone()), ("schema", schema.clone())], + )? + .extract()?; + if let Some(source_names) = source_names { + if source_names.is_empty() { + return empty_set(schema); + } + read_sql = relevant_sources_sql(&read_sql, source_names); + } + let result: Bound<'py, PyAny> = match call_with( + &read.execute, + &[ + ("connection", read.connection.clone()), + ("sql", text(py, &read_sql)), + ], + ) { + Ok(result) => result, + Err(error) => { + if exception_text(py, &error)?.is_none() { + return Err(error); + } + return Err(freshness_error( + py, + ( + INPUT_ERROR, + &format!( + "Unable to read source freshness records from {qualified_name}. This can \ +happen after upgrading from an older sqlbuild version; delete or rebuild the SQLBuild source \ +freshness table to regenerate source freshness state." + ), + ), + Some(error), + )); + } + }; + let records: Bound<'py, PyDict> = PyDict::new(py); + for row in result.call_method0("fetchall")?.try_iter()? { + let record: Bound<'py, PyAny> = row_record(&row?)?; + records.set_item(record.getattr("identity")?, record)?; + } + call_with( + &imported(py, FRESHNESS_MODELS, "SourceFreshnessSet")?, + &[("schema", schema.clone()), ("records", records.into_any())], + ) +} + +fn row_record<'py>(row: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = row.py(); + let raw_observed_at: Bound<'py, PyAny> = row.get_item(9)?; + let observed_at: Bound<'py, PyAny> = if date_time_typed(&raw_observed_at)? { + raw_observed_at + } else { + imported(py, DATETIME_MODULE, "datetime")? + .call_method1("fromisoformat", (raw_observed_at.str()?,))? + }; + let observed_at: Bound<'py, PyAny> = presumed_utc_datetime(&observed_at)?; + let field = |index: usize| -> PyResult> { + Ok(row.get_item(index)?.str()?.into_any()) + }; + let optional = + |index: usize| -> PyResult> { optional_str(&row.get_item(index)?) }; + call_with( + &imported(py, FRESHNESS_MODELS, "SourceFreshnessRecord")?, + &[ + ("source_name", field(0)?), + ("target_database", optional(1)?), + ("target_schema", optional(2)?), + ("target_name", optional(3)?), + ("run_id", field(4)?), + ("strategy", field(5)?), + ("value_kind", field(6)?), + ("data_version", optional(7)?), + ("data_version_hash", field(8)?), + ("observed_at", observed_at), + ], + ) +} + +/// Read latest source freshness rows, trusting the caller-resolved table existence. +#[pyfunction] +#[pyo3(signature = (**options))] +fn read_latest_source_freshness<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "read_latest_source_freshness", + options, + ( + &[ + "connection", + "execute", + "table_exists", + "database", + "schema", + "render_qualified_name", + "render_read_latest_sql", + ], + &["source_names"], + ), + )?; + let read: StateRead<'py> = StateRead { + connection: arguments.value("connection")?, + execute: arguments.value("execute")?, + render_qualified_name: arguments.value("render_qualified_name")?, + render_read_latest_sql: arguments.value("render_read_latest_sql")?, + }; + let source_names: Option> = arguments.value("source_names")?.extract()?; + read_latest( + &read, + (&arguments.value("database")?, &arguments.value("schema")?), + arguments.value("table_exists")?.is_truthy()?, + source_names.as_deref(), + ) +} + +fn execute_sql( + (execute, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + sql: Bound<'_, PyAny>, +) -> PyResult<()> { + call_with(execute, &[("connection", connection.clone()), ("sql", sql)])?; + Ok(()) +} + +/// Append source freshness rows, creating the table once if needed. +#[pyfunction] +#[pyo3(signature = (**options))] +fn write_source_freshness_records<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult<()> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "write_source_freshness_records", + options, + ( + &[ + "connection", + "execute", + "database", + "schema", + "records", + "renderers", + ], + &[], + ), + )?; + let connection: &Bound<'py, PyAny> = &arguments.value("connection")?; + let execute: &Bound<'py, PyAny> = &arguments.value("execute")?; + let database: &Bound<'py, PyAny> = &arguments.value("database")?; + let schema: &Bound<'py, PyAny> = &arguments.value("schema")?; + let records: &Bound<'py, PyAny> = &arguments.value("records")?; + let renderers: &Bound<'py, PyAny> = &arguments.value("renderers")?; + if !records.is_truthy()? { + return Ok(()); + } + let location = || -> Vec<(&str, Bound<'py, PyAny>)> { + vec![("database", database.clone()), ("schema", schema.clone())] + }; + let render_create: Bound<'py, PyAny> = renderers.getattr("render_create_table_sql")?; + let create_sql: Bound<'py, PyAny> = if render_create.is_none() { + create_table_sql( + (database, schema), + &renderers.getattr("render_qualified_name")?, + &renderers.getattr("render_framework_type")?, + )? + } else { + call_with(&render_create, &location())? + }; + execute_sql((execute, connection), create_sql)?; + let render_indexes: Bound<'py, PyAny> = renderers.getattr("render_create_index_sqls")?; + if !render_indexes.is_none() { + for index_sql in call_with(&render_indexes, &location())?.try_iter()? { + execute_sql((execute, connection), index_sql?)?; + } + } + let mut insert: Vec<(&str, Bound<'py, PyAny>)> = location(); + insert.push(("records", records.clone())); + let insert_sql: Bound<'py, PyAny> = + call_with(&renderers.getattr("render_insert_records_sql")?, &insert)?; + execute_sql((execute, connection), insert_sql) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_source_freshness_read_latest_sql, + module + )?)?; + module.add_function(wrap_pyfunction!( + build_source_freshness_create_table_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(read_latest_source_freshness, module)?)?; + module.add_function(wrap_pyfunction!(write_source_freshness_records, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs new file mode 100644 index 0000000000..610d3e5aeb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs @@ -0,0 +1,81 @@ +//! Python objects, exceptions and datetimes the source freshness glue shares. + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyString}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; +use crate::bindings::_helpers::state_rows::timestamps::{date_time_typed, in_utc, offset_aware}; + +pub(crate) const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; +pub(crate) const FRESHNESS_MODELS: &str = "sqlbuild.compiler.source_freshness.models"; +pub(crate) const FRESHNESS_TYPES: &str = "sqlbuild.compiler.source_freshness.types"; +pub(crate) const FRESHNESS_CONSTANTS: &str = "sqlbuild.compiler.source_freshness.constants"; +pub(crate) const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; +pub(crate) const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; +pub(crate) const ADAPTER_MODELS: &str = "sqlbuild.adapter.contract.models"; +pub(crate) const ADAPTER_TYPES: &str = "sqlbuild.adapter.contract.types"; +pub(crate) const ADAPTER_EXCEPTIONS: &str = "sqlbuild.adapter.contract.exceptions"; +pub(crate) const LIFECYCLE_MODULE: &str = + "sqlbuild.runtime.observability.classes.operation_lifecycle"; +pub(crate) const OBSERVABILITY_MODELS: &str = "sqlbuild.runtime.observability.models"; +pub(crate) const STATE_TABLE_DDL_MODULE: &str = + "sqlbuild.adapter.state_sql.main.render_state_table_create_sql"; +pub(crate) const OBSERVATION_ERROR: &str = "SourceFreshnessObservationError"; +pub(crate) const INPUT_ERROR: &str = "SourceFreshnessInputError"; + +/// `exceptions.(message)` as a raisable error, with an optional `__cause__`. +pub(crate) fn freshness_error( + py: Python<'_>, + (class, message): (&str, &str), + cause: Option, +) -> PyErr { + let error: PyErr = + match imported(py, EXCEPTIONS_MODULE, class).and_then(|class| class.call1((message,))) { + Ok(error) => PyErr::from_value(error), + Err(error) => return error, + }; + if let Some(cause) = cause { + error.set_cause(py, Some(cause)); + } + error +} + +/// `str(exc)` of an `Exception`, or the error itself for anything else. +pub(crate) fn exception_text(py: Python<'_>, error: &PyErr) -> PyResult> { + if !error.is_instance_of::(py) { + return Ok(None); + } + Ok(Some(error.value(py).str()?.extract()?)) +} + +/// `value` when it is not a `datetime`; otherwise in UTC, a naive value read as UTC. +pub(crate) fn presumed_utc_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if !date_time_typed(value)? { + return Ok(value.clone()); + } + presumed_utc_datetime(value) +} + +/// A `datetime` in UTC, a naive or offset-less value read as UTC. +pub(crate) fn presumed_utc_datetime<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if offset_aware(value)? { + return in_utc(value); + } + let changes: Bound<'py, PyDict> = PyDict::new(value.py()); + changes.set_item("tzinfo", imported(value.py(), DATETIME_MODULE, "UTC")?)?; + value.call_method("replace", (), Some(&changes)) +} + +pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +/// `str(value)`, or `None` for `None`. +pub(crate) fn optional_str<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(value.clone()); + } + Ok(value.str()?.into_any()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/cache_directory.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/cache_directory.rs new file mode 100644 index 0000000000..a24bff111a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/cache_directory.rs @@ -0,0 +1,35 @@ +//! The project-local compiler store root. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const FRONTIER_CONSTANTS_MODULE: &str = "sqlbuild.compiler.frontier.constants"; + +/// `project_dir / target / cache / compiler`, keeping the path type of `project_dir`. +pub(crate) fn cache_directory<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let constant = |name: &str| -> PyResult { + imported(py, FRONTIER_CONSTANTS_MODULE, name)?.extract() + }; + let store: String = format!( + "{}{}", + constant("COMPILER_CACHE_DIRECTORY_NAME")?, + constant("NATIVE_CACHE_NAMESPACE_SUFFIX")? + ); + project_dir + .div(constant("TARGET_DIRECTORY_NAME")?)? + .div(constant("CACHE_DIRECTORY_NAME")?)? + .div(store) +} + +/// Return the project's compiler store root. +#[pyfunction] +fn compiler_cache_directory<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + cache_directory(project_dir) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(compiler_cache_directory, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs index fa17bbe460..d752ee27cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod cache_directory; pub(crate) mod compile_frontier; pub(crate) mod encoder; pub(crate) mod python_types; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/registration.rs index 82b9d86c7e..de98b71172 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::frontier::{compile_frontier, stage_capture}; +use crate::bindings::_helpers::frontier::{cache_directory, compile_frontier, stage_capture}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + cache_directory::register(module)?; compile_frontier::register(module)?; stage_capture::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index b477e79fbd..1565112d7f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -35,8 +35,7 @@ const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; -const CURSOR_OVERRIDES_MODULE: &str = - "sqlbuild.compiler.planner.main.cursor_window._typed_cursor_overrides"; +const CURSOR_OVERRIDES_MODULE: &str = "sqlbuild._native"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const PROVIDER_SESSION_MODULE: &str = "sqlbuild.provider.main.session"; const PYTHON_NODE_MODELS: &str = "sqlbuild.python_nodes.models"; @@ -324,7 +323,7 @@ fn prepare<'py>( requested.extract()? }; let typed: Bound<'py, PyAny> = call_with( - &imported(py, CURSOR_OVERRIDES_MODULE, "typed_cursor_overrides")?, + &imported(py, CURSOR_OVERRIDES_MODULE, "type_cursor_overrides")?, &[("cursor_overrides", request.getattr("cursor_overrides")?)], )?; let cursor_overrides: Bound<'py, PyAny> = if typed.is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs index 506e5e996c..1ae444823e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::build_progress::terminal::start_spinner; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const LINES_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; +const LINES_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_TYPES: &str = "sqlbuild.adapter.contract.types"; const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; const SPINNER_TICK_SECONDS: f64 = 0.1; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index e7a77b676b..33931214d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -21,6 +21,7 @@ pub(crate) mod contract_command; pub(crate) mod contracts; pub(crate) mod cost_command; pub(crate) mod cost_report; +pub(crate) mod cost_telemetry; pub(crate) mod cursor; pub(crate) mod dag_artifact; pub(crate) mod dbt_command; @@ -46,6 +47,7 @@ pub(crate) mod model_config; pub(crate) mod plan_command; pub(crate) mod plan_output; pub(crate) mod planner_changes; +pub(crate) mod planner_entries; pub(crate) mod planner_identity; pub(crate) mod planner_migrations; pub(crate) mod planner_phases; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs index ee7d58978a..4727631155 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs @@ -1,11 +1,19 @@ //! Native config errors handed to Python, which raises them as the Python stages would. -use pyo3::Py; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{Py, PyErr, pymethods}; use sqlbuild_model_config::errors::{ConfigError, ErrorClass}; +use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::model_config::errors::NativeConfigError; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const IDENTITY_ERRORS_MODULE: &str = "sqlbuild.compiler.resource_names.exceptions"; +const CONTRACT_ERRORS_MODULE: &str = "sqlbuild.spec.contracts.exceptions"; +const EXPECTED_STRING: &str = "a string"; + const COMPILE_INPUT_CLASS: &str = "compile_input"; const CONFIG_VALUE_TYPE_CLASS: &str = "config_value_type"; const RESOURCE_IDENTITY_CLASS: &str = "resource_identity"; @@ -34,6 +42,59 @@ pub(crate) fn native_config_error( ) } +/// The Python exception a native config error raises as, given the values it names. +pub(crate) fn config_exception<'py>( + error: &Bound<'py, PyAny>, + values: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let error: Bound<'py, NativeConfigError> = error.cast::()?.clone(); + error.get().exception(error.py(), values) +} + +/// `config_exception` as the raised error, or the failure building it. +pub(crate) fn raised_config_exception( + error: &Bound<'_, PyAny>, + values: Option<&Bound<'_, PyAny>>, +) -> PyErr { + match config_exception(error, values) { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +#[pymethods] +impl NativeConfigError { + /// The exception this error raises as: its class, message, code and help. + #[pyo3(signature = (values = None))] + fn exception<'py>( + &self, + py: Python<'py>, + values: Option<&Bound<'py, PyAny>>, + ) -> PyResult> { + if self.class_name == CONFIG_VALUE_TYPE_CLASS + && let (Some(key), Some(values)) = (&self.key, values) + { + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("key", key)?; + keywords.set_item("expected", EXPECTED_STRING)?; + keywords.set_item("actual_type", values.get_item(key)?.get_type())?; + return imported(py, CONTRACT_ERRORS_MODULE, "ConfigValueTypeError")? + .call((), Some(&keywords)); + } + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("help", &self.help)?; + let (module, class) = match self.class_name { + RESOURCE_IDENTITY_CLASS => (IDENTITY_ERRORS_MODULE, "ResourceIdentityError"), + DISCOVERY_CONFLICT_CLASS => (DISCOVERY_ERRORS_MODULE, "DiscoveryConflictError"), + _ => { + keywords.set_item("code", self.code)?; + (COMPILE_ERRORS_MODULE, "CompileInputError") + } + }; + imported(py, module, class)?.call((&self.message,), Some(&keywords)) + } +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs new file mode 100644 index 0000000000..1cca55b389 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs @@ -0,0 +1,153 @@ +//! The Python classes native header and YAML parsing build contract objects as. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const AUDITING_MODELS_MODULE: &str = "sqlbuild.compiler.auditing.models"; +const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +const DLT_MODELS_MODULE: &str = "sqlbuild.integrations.dlt.models"; +const INGESTR_MODELS_MODULE: &str = "sqlbuild.integrations.ingestr.models"; +const CONTRACT_CLASSES: [(&str, &str, &str); 6] = [ + ("schema_column", CONTRACT_MODELS_MODULE, "SchemaColumn"), + ( + "schema_audit_instance", + CONTRACT_MODELS_MODULE, + "SchemaAuditInstance", + ), + ( + "schema_dynamic_column_family", + CONTRACT_MODELS_MODULE, + "SchemaDynamicColumnFamily", + ), + ("source_location", CONTRACT_MODELS_MODULE, "SourceLocation"), + ( + "measurement_thresholds", + AUDITING_MODELS_MODULE, + "MeasurementThresholds", + ), + ( + "measurement_threshold_bound", + AUDITING_MODELS_MODULE, + "MeasurementThresholdBound", + ), +]; +const YAML_ENTRY_CLASSES: [(&str, &str, &str); 13] = [ + ( + "schema_seed_entry", + CONTRACT_MODELS_MODULE, + "SchemaSeedEntry", + ), + ( + "seed_csv_settings", + CONTRACT_MODELS_MODULE, + "SeedCsvSettings", + ), + ("source_entry", CONTRACT_MODELS_MODULE, "SourceEntry"), + ( + "source_column_entry", + CONTRACT_MODELS_MODULE, + "SourceColumnEntry", + ), + ( + "source_freshness_config", + CONTRACT_MODELS_MODULE, + "SourceFreshnessConfig", + ), + ( + "source_freshness_age_policy", + CONTRACT_MODELS_MODULE, + "SourceFreshnessAgePolicy", + ), + ( + "integration_loader_config", + CONTRACT_MODELS_MODULE, + "IntegrationLoaderConfig", + ), + ( + "ingestr_source_config", + INGESTR_MODELS_MODULE, + "IngestrSourceConfig", + ), + ("dlt_source_config", DLT_MODELS_MODULE, "DltSourceConfig"), + ( + "dlt_resource_config", + DLT_MODELS_MODULE, + "DltResourceConfig", + ), + ( + "source_write_strategy", + CONTRACT_TYPES_MODULE, + "SourceWriteStrategy", + ), + ( + "source_freshness_strategy", + CONTRACT_TYPES_MODULE, + "SourceFreshnessStrategy", + ), + ( + "source_freshness_value_kind", + CONTRACT_TYPES_MODULE, + "SourceFreshnessValueKind", + ), +]; + +static CONTRACT_CLASS_TABLE: PyOnceLock> = PyOnceLock::new(); +static YAML_ENTRY_CLASS_TABLE: PyOnceLock> = PyOnceLock::new(); + +/// The contract classes native header parsing builds columns and audits as. +pub(crate) fn contract_classes(py: Python<'_>) -> PyResult> { + CONTRACT_CLASS_TABLE + .get_or_try_init(py, || built_contract_classes(py).map(Bound::unbind)) + .map(|classes| classes.bind(py).clone()) +} + +/// The classes parsed seed and source declarations become. +pub(crate) fn yaml_entry_classes(py: Python<'_>) -> PyResult> { + YAML_ENTRY_CLASS_TABLE + .get_or_try_init(py, || built_yaml_entry_classes(py).map(Bound::unbind)) + .map(|classes| classes.bind(py).clone()) +} + +fn built_contract_classes(py: Python<'_>) -> PyResult> { + let classes: Bound<'_, PyDict> = PyDict::new(py); + for (key, module, name) in CONTRACT_CLASSES { + classes.set_item(key, imported(py, module, name)?)?; + } + classes.set_item( + "severities", + members_by_value(&imported(py, AUDITING_TYPES_MODULE, "AuditSeverity")?)?, + )?; + classes.set_item( + "threshold_operators", + members_by_value(&imported(py, AUDITING_TYPES_MODULE, "ThresholdOperator")?)?, + )?; + classes.set_item( + "allocate", + imported(py, "builtins", "object")?.getattr("__new__")?, + )?; + Ok(classes) +} + +fn built_yaml_entry_classes(py: Python<'_>) -> PyResult> { + let classes: Bound<'_, PyDict> = PyDict::new(py); + classes.set_item("contracts", contract_classes(py)?)?; + for (key, module, name) in YAML_ENTRY_CLASSES { + classes.set_item(key, imported(py, module, name)?)?; + } + Ok(classes) +} + +/// `{member.value: member for member in enum}`. +fn members_by_value<'py>(enumeration: &Bound<'py, PyAny>) -> PyResult> { + let members: Bound<'py, PyDict> = PyDict::new(enumeration.py()); + for member in enumeration.try_iter()? { + let member: Bound<'py, PyAny> = member?; + members.set_item(member.getattr("value")?, member)?; + } + Ok(members) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs index c7f002ec28..3c8b08c2a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs @@ -10,14 +10,13 @@ use sqlbuild_model_config::header_metadata::main::parse_schema_columns::parse_sc use sqlbuild_model_config::header_metadata::models::ColumnRequest; use crate::bindings::_helpers::boundary::panics::compiler_guard; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::model_config::config_errors::config_exception; use crate::bindings::_helpers::model_config::config_errors::native_config_error; +use crate::bindings::_helpers::model_config::contract_classes::contract_classes; use crate::bindings::_helpers::model_config::header_metadata::{ContractClasses, column_tuple}; -const CONTRACT_CLASSES_MODULE: &str = - "sqlbuild.compiler.model_config.main._native_contract_classes"; -const CONFIG_ERROR_MODULE: &str = "sqlbuild.compiler.model_config.main._native_config_error"; const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; const DECLARATION_ERROR_CLASS: &str = "DeclarationParseError"; @@ -44,10 +43,7 @@ pub(crate) fn schema_columns<'py>( }; let columns = parse_columns(&PyNode(raw_columns.clone()), request) .map_err(|error| raised(py, error, declaration_errors))?; - let classes: ContractClasses<'py> = - imported(py, CONTRACT_CLASSES_MODULE, "native_contract_classes")? - .call0()? - .extract()?; + let classes: ContractClasses<'py> = contract_classes(py)?.extract()?; column_tuple(py, &classes, &columns, column_locations) } @@ -70,10 +66,7 @@ fn raised(py: Python<'_>, error: ConfigError, declaration_errors: bool) -> PyErr .call1((error.message,)); } let native = native_config_error(py, error)?.into_bound(py).into_any(); - call_with( - &imported(py, CONFIG_ERROR_MODULE, "native_config_error")?, - &[("error", native)], - ) + config_exception(&native, None) })(); match built { Ok(instance) => PyErr::from_value(instance), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index d76eb0322f..0d0f9cbb49 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod column_migrations; pub(crate) mod config_errors; pub(crate) mod config_templates; pub(crate) mod config_values; +pub(crate) mod contract_classes; pub(crate) mod described_models; pub(crate) mod errors; pub(crate) mod header_columns; @@ -12,3 +13,4 @@ pub(crate) mod model_validator; pub(crate) mod project_defaults; pub(crate) mod registration; pub(crate) mod template_data; +pub(crate) mod unicode_environment; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index 4e219ebeab..5ee6943fc6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -22,6 +22,7 @@ use sqlbuild_model_config::types::AuthoredNode; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compiled_project::model_loop_config::{ConfigClasses, built_config}; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::model_config::config_errors::config_exception; use crate::bindings::_helpers::model_config::config_errors::native_config_error; use crate::bindings::_helpers::model_config::config_templates::{ PythonHost, Stop, TemplateSources, expanded, template_error, @@ -36,8 +37,6 @@ const MODEL_CONFIG_LABEL: &str = "model config"; const ENVIRONMENT_DATABASE_LABEL: &str = "environment database"; const ENVIRONMENT_SCHEMA_LABEL: &str = "environment schema"; const TAGS_KEY: &str = "tags"; -const NATIVE_CONFIG_ERROR_MODULE: &str = "sqlbuild.compiler.model_config.main._native_config_error"; -const NATIVE_CONFIG_ERROR_FUNCTION: &str = "native_config_error"; const ROW_DIFF_EXCLUDE_COLUMNS_KEY: &str = "row_diff_exclude_columns"; const ROW_DIFF_TOLERANCES_KEY: &str = "row_diff_tolerances"; const AUDIT_OVERRIDE_SECTIONS: [&str; 2] = ["by_type", "by_column"]; @@ -586,17 +585,14 @@ pub(crate) fn layered_values<'py>( }) } -/// The Python exception `native_config_error` makes of a native config error. +/// The Python exception a native config error raises as. pub(crate) fn raised_config_error(py: Python<'_>, error: ConfigError) -> PyErr { let raised = || -> PyResult { let error: Py = native_config_error(py, error)?; - let arguments: Bound<'_, PyDict> = PyDict::new(py); - arguments.set_item("error", error)?; - let exception: Bound<'_, PyAny> = py - .import(NATIVE_CONFIG_ERROR_MODULE)? - .getattr(NATIVE_CONFIG_ERROR_FUNCTION)? - .call((), Some(&arguments))?; - Ok(PyErr::from_value(exception)) + Ok(PyErr::from_value(config_exception( + error.bind(py).as_any(), + None, + )?)) }; raised().unwrap_or_else(|error| error) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs index 8f9293b80f..fc8b3615f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::model_config::{ column_migrations, config_errors, config_values, described_models, header_columns, - model_config_builder, model_validator, template_data, + model_config_builder, model_validator, template_data, unicode_environment, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -16,5 +16,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { model_config_builder::register(module)?; model_validator::register(module)?; template_data::register(module)?; + unicode_environment::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs index 28dedd70e0..dcd165dfa1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs @@ -11,16 +11,16 @@ use sqlbuild_core::scenario_command::main::format_captured_at::format_captured_a use sqlbuild_model_config::templates::models::TemplateOptions; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compile_driver::input_reads::{context_read, environment_read}; +use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; +use crate::bindings::_helpers::model_config::config_errors::config_exception; use crate::bindings::_helpers::model_config::config_templates::{ TemplateOutcome, TemplateSources, expand_config_templates, lone_surrogate_site, resolve_effective_vars, }; +use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; -const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; -const CONFIG_ERROR_MODULE: &str = "sqlbuild.compiler.model_config.main._native_config_error"; const UUID_MODULE: &str = "uuid"; const ENVIRONMENT_READ: &str = "env"; const RUN_ID_KEY: &str = "run.id"; @@ -34,14 +34,12 @@ pub(crate) fn record_reads<'a>( py: Python<'_>, reads: impl IntoIterator, ) -> PyResult<()> { - let registry: Bound<'_, PyAny> = imported(py, COMPILE_CONSTANTS_MODULE, "COMPILE_INPUT_READS")?; for (kind, name) in reads { - let method: &str = if kind == ENVIRONMENT_READ { - "environment_read" + if kind == ENVIRONMENT_READ { + environment_read(py, name)?; } else { - "context_read" - }; - registry.call_method1(method, (name,))?; + context_read(py, name)?; + } } Ok(()) } @@ -52,16 +50,16 @@ fn applied<'py>(py: Python<'py>, outcome: TemplateOutcome<'py>) -> PyResult Ok(value), - Err(error) => Err(PyErr::from_value(call_with( - &imported(py, CONFIG_ERROR_MODULE, "native_config_error")?, - &[("error", error.into_bound(py).into_any())], + Err(error) => Err(PyErr::from_value(config_exception( + error.bind(py).as_any(), + None, )?)), } } /// The process environment as compile text reads it. fn environment(py: Python<'_>) -> PyResult> { - imported(py, UNICODE_ENVIRONMENT_MODULE, "UnicodeEnvironment")?.call0() + unicode_environment(py) } /// `value` with every `${...}` template expanded, recording the reads it made. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs new file mode 100644 index 0000000000..5a2970e1dd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs @@ -0,0 +1,79 @@ +//! The process environment as compile text reads it, rejecting values that are not UTF-8. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyString}; +use pyo3::{pyclass, pymethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; + +/// `os.environ`, raising a clear error for a value holding undecodable bytes. +#[pyclass(module = "sqlbuild._native", frozen, mapping)] +pub(crate) struct UnicodeEnvironment; + +fn environ(py: Python<'_>) -> PyResult> { + imported(py, "os", "environ") +} + +#[pymethods] +impl UnicodeEnvironment { + #[new] + fn new() -> Self { + Self + } + + fn __getitem__<'py>(&self, name: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = name.py(); + let value: Bound<'py, PyAny> = environ(py)?.get_item(name)?; + let encoded: Bound<'py, PyBytes> = value + .cast::()? + .call_method1("encode", ("utf-8", "surrogatepass"))? + .cast_into::()?; + let byte_offset: usize = match std::str::from_utf8(encoded.as_bytes()) { + Ok(_) => return Ok(value), + Err(error) => error.valid_up_to(), + }; + let name: String = name.str()?.to_string(); + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item( + "help", + format!( + "Set '{name}' to UTF-8 text, since SQLBuild cannot send undecodable bytes to a \ + warehouse; the value is not shown because it may be a secret" + ), + )?; + Err(PyErr::from_value( + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?.call( + (format!( + "Environment variable '{name}' is not valid UTF-8 text: byte {byte_offset} \ + starts an invalid UTF-8 sequence" + ),), + Some(&keywords), + )?, + )) + } + + fn __contains__(&self, name: &Bound<'_, PyAny>) -> PyResult { + environ(name.py())?.contains(name) + } + + fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + environ(py)?.try_iter().map(Bound::into_any) + } + + fn __len__(&self, py: Python<'_>) -> PyResult { + environ(py)?.len() + } +} + +/// A new view of the process environment as compile text reads it. +pub(crate) fn unicode_environment(py: Python<'_>) -> PyResult> { + Ok(Bound::new(py, UnicodeEnvironment)?.into_any()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs index 4e2f85dae8..f835c7cf12 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs @@ -8,9 +8,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cli_text::models::{ CliStyle, ColumnMigrationText, MigrationDecisionText, MigrationText, MigrationTextReport, - OldNameText, + OldNameText, PreBuildWorkText, RetentionText, TableTypeText, +}; +use sqlbuild_core::plan_display::main::{ + format_model_migrations, format_pre_build_work, old_name_rows, }; -use sqlbuild_core::plan_display::main::{format_model_migrations, old_name_rows}; use std::collections::BTreeMap; fn label(value: &Bound<'_, PyAny>) -> PyResult> { @@ -85,8 +87,8 @@ fn column(entry: &Bound<'_, PyAny>) -> PyResult { }) } -#[pyfunction(name = "format_model_migrations", signature = (*, lines, plan, display_options))] -fn format_migrations( +/// Migration sections appended to `lines`, in planner order. +pub(crate) fn migration_lines( lines: Vec, plan: &Bound<'_, PyAny>, display_options: &Bound<'_, PyAny>, @@ -125,6 +127,79 @@ fn format_migrations( )) } +#[pyfunction(name = "format_model_migrations", signature = (*, lines, plan, display_options))] +fn format_migrations( + lines: Vec, + plan: &Bound<'_, PyAny>, + display_options: &Bound<'_, PyAny>, +) -> PyResult> { + migration_lines(lines, plan, display_options) +} + +fn days(value: &Bound<'_, PyAny>) -> PyResult> { + value.extract() +} + +/// Table-type, retention and migration work that runs before model builds. +pub(crate) fn pre_build_lines( + lines: Vec, + plan: &Bound<'_, PyAny>, + display_options: &Bound<'_, PyAny>, +) -> PyResult> { + let mut table_types: Vec = Vec::new(); + for entry in items(&plan.getattr("table_type_entries")?)? { + table_types.push(TableTypeText { + model_name: text(&entry.getattr("model_name")?)?, + desired_type: text(&entry.getattr("desired_type")?)?, + actual_type: optional_text(&entry.getattr("actual_type")?)? + .filter(|actual| !actual.is_empty()), + source: text(&entry.getattr("source")?)?, + irreversible_warning: optional_text(&entry.getattr("irreversible_warning")?)?, + }); + } + let mut retention: Vec = Vec::new(); + for entry in items(&plan.getattr("retention_entries")?)? { + let request = entry.getattr("request")?; + let mut parts: Vec = Vec::new(); + for field in ["database", "schema", "name"] { + let part = request.getattr(field)?; + if part.is_truthy()? { + parts.push(text(&part)?); + } + } + retention.push(RetentionText { + scope: parts.join("."), + desired_days: request.getattr("desired_days")?.extract()?, + actual_days: days(&entry.getattr("actual_days")?)?, + effective_days: days(&entry.getattr("effective_days")?)?, + source: text(&entry.getattr("source")?)?, + direction: text(&entry.getattr("direction")?.getattr("value")?)?, + phase: text(&entry.getattr("phase")?.getattr("value")?)?, + irreversible_warning: optional_text(&entry.getattr("irreversible_warning")?)?, + }); + } + let work = PreBuildWorkText { + table_types, + retention, + display: display_limit(display_options)?, + }; + migration_lines( + format_pre_build_work::format_pre_build_work(lines, &work), + plan, + display_options, + ) +} + +/// Format retention, table-type, and migration work that runs before model builds. +#[pyfunction(name = "format_pre_build_work", signature = (*, lines, plan, display_options))] +fn format_pre_build( + lines: Vec, + plan: &Bound<'_, PyAny>, + display_options: &Bound<'_, PyAny>, +) -> PyResult> { + pre_build_lines(lines, plan, display_options) +} + #[pyfunction(name = "old_name_rows", signature = (*, entry, style))] fn render_old_name(entry: &Bound<'_, PyAny>, style: &Bound<'_, PyAny>) -> PyResult> { Ok(old_name_rows::old_name_rows( @@ -137,5 +212,6 @@ fn render_old_name(entry: &Bound<'_, PyAny>, style: &Bound<'_, PyAny>) -> PyResu pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(format_migrations, module)?)?; + module.add_function(wrap_pyfunction!(format_pre_build, module)?)?; module.add_function(wrap_pyfunction!(render_old_name, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs index 51239c58ac..123d07fb5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs @@ -362,7 +362,11 @@ fn warning_json<'py>(warning: &Bound<'py, PyAny>) -> PyResult> } /// `difflib.unified_diff` between two texts, each line without its newline. -fn unified_diff<'py>(py: Python<'py>, previous: &str, current: &str) -> PyResult> { +pub(crate) fn unified_diff<'py>( + py: Python<'py>, + previous: &str, + current: &str, +) -> PyResult> { let lines = |text: &str| -> PyResult> { PyString::new(py, text).call_method1("splitlines", (true,)) }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs index e4377d901e..d2ebdc8e5f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs @@ -6,14 +6,13 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cli_text::main::format_plan_completion::format_plan_completion; use sqlbuild_core::cli_text::main::format_plan_text::format_plan_text; use sqlbuild_core::cli_text::models::{ - CursorPlanText, DisplayLimit, FunctionPlanText, ModelPlanText, PlanTextReport, PlanWarningText, - ProviderUsageText, PythonPlanText, RenameHintText, SchemaFindingText, SeedPlanText, - SourceFreshnessText, SourceLoadPlanText, + CliStyle, CursorPlanText, DisplayLimit, FunctionPlanText, ModelPlanText, PlanTextReport, + PlanWarningText, ProviderUsageText, PythonPlanText, RenameHintText, SchemaFindingText, + SeedPlanText, SourceFreshnessText, SourceLoadPlanText, }; +use sqlbuild_core::plan_display::main::format_query_diff::format_query_diff; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::plan_output::cursor_details::{ CursorDetails, DEFERRED, RESOLVED, cursor_details, runtime_inputs_clause, @@ -22,13 +21,13 @@ use crate::bindings::_helpers::plan_output::display::{ cursor_bound_display, format_no_input_rows, format_waiting_on_empty_inputs, inclusive_cursor_end, model_materialization_label, }; +use crate::bindings::_helpers::plan_output::migrations::pre_build_lines; +use crate::bindings::_helpers::plan_output::plan_json::unified_diff; use crate::bindings::_helpers::plan_output::python_identity::{ definition_source_text, dependency_source_text, }; use crate::bindings::_helpers::plan_output::values::{items, optional_text, text, texts}; -const QUERY_DIFF_MODULE: &str = "sqlbuild.compiler.planner.main.changes._query_diff"; -const PRE_BUILD_MODULE: &str = "sqlbuild.compiler.planner.main.pre_build.pre_build_work_display"; const DISPLAY_MODELS_MODULE: &str = "sqlbuild.presentation.models"; const SKIP_ACTION: &str = "skip"; const QUERY_CHANGED: &str = "query_changed"; @@ -48,16 +47,17 @@ const PLAN_OPTIONS: (&[&str], &[&str]) = ( ], ); -/// `format_query_diff(previous=..., current=...)`. +/// The coloured unified diff between a previous and a current query. pub(crate) fn query_diff( previous: &Bound<'_, PyAny>, current: &Bound<'_, PyAny>, ) -> PyResult> { - call_with( - &imported(previous.py(), QUERY_DIFF_MODULE, "format_query_diff")?, - &[("previous", previous.clone()), ("current", current.clone())], - )? - .extract() + let lines = unified_diff( + previous.py(), + &previous.extract::()?, + ¤t.extract::()?, + )?; + Ok(format_query_diff(&lines, CliStyle { use_color: true })) } fn token<'py>(bound: Bound<'py, PyAny>) -> PyResult> { @@ -437,14 +437,7 @@ fn report(arguments: &KeywordArguments<'_>) -> PyResult { reason: text(&seed.getattr("reason")?)?, }); } - let pre_build: Bound<'_, PyAny> = call_with( - &imported(py, PRE_BUILD_MODULE, "format_pre_build_work")?, - &[ - ("lines", PyList::empty(py).into_any()), - ("plan", plan.clone()), - ("display_options", display_options.clone()), - ], - )?; + let pre_build: Vec = pre_build_lines(Vec::new(), &plan, &display_options)?; Ok(PlanTextReport { use_color: flag_or("use_color", true)?, include_header: flag_or("include_header", true)?, @@ -476,7 +469,7 @@ fn report(arguments: &KeywordArguments<'_>) -> PyResult { metadata.call_method1("get", ("direct_remaining_stale_model_names",))?, ))?, pruned_models: pruned_models(&metadata)?, - pre_build_lines: pre_build.extract()?, + pre_build_lines: pre_build, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs new file mode 100644 index 0000000000..514364147f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs @@ -0,0 +1,661 @@ +//! Clone planning: destination and origin entries, and deferred-clone graph traversal. + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::planner_entries::values::{ + COMPILE_MODELS, native, planner_model, planner_type, string, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +const VIEW_MATERIALIZATION: &str = "view"; +const GRAPH_ARGUMENTS: [&str; 4] = ["selected", "upstream", "is_clonable", "is_view"]; + +/// Whether a model is explicitly disabled with a boolean `enabled = false`. +fn disabled_model(model: &Bound<'_, PyAny>) -> PyResult { + let raw = model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("enabled",))?; + Ok(raw.is_instance_of::() && !raw.is_truthy()?) +} + +fn tuple_of<'py>(py: Python<'py>, items: Vec>) -> PyResult> { + Ok(PyTuple::new(py, items)?.into_any()) +} + +/// Entries keyed by node key, returned in `execution_order`. +fn in_order<'py>( + entries: &Bound<'py, PyDict>, + execution_order: &Bound<'py, PyAny>, +) -> PyResult> { + let mut ordered: Vec> = Vec::new(); + for key in execution_order.try_iter()? { + if let Some(entry) = entries.get_item(key?)? { + ordered.push(entry); + } + } + tuple_of(entries.py(), ordered) +} + +fn clone_plan_output<'py>( + project: &Bound<'py, PyAny>, + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let upstream = + imported(py, "sqlbuild._native", "build_execution_upstream_deps")?.call1((project,))?; + let downstream = + imported(py, "sqlbuild._native", "build_downstream_deps")?.call1((&upstream,))?; + let selected = native( + py, + "resolve_graph_selection", + &[ + ("graph", project.getattr("lineage_graph")?), + ("select", select.clone()), + ("exclude", exclude.clone()), + ], + )?; + planner_model( + py, + "PlanOutput", + &[ + ( + "execution_order", + native( + py, + "topologically_order_keys", + &[("upstream", upstream.clone())], + )?, + ), + ("selected_keys", selected), + ("upstream_deps", upstream), + ("downstream_deps", downstream), + ], + ) +} + +/// Locations and source columns that destination SQL resolves against. +struct CloneResolution<'py> { + model_locations: Bound<'py, PyAny>, + seed_locations: Bound<'py, PyAny>, + function_locations: Bound<'py, PyAny>, + source_map: Bound<'py, PyAny>, + source_warehouse_columns: Bound<'py, PyAny>, +} + +fn clone_resolution<'py>( + project: &Bound<'py, PyAny>, + source_project: &Bound<'py, PyAny>, + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py = project.py(); + let model_locations = native( + py, + "build_model_locations", + &[("models", project.getattr("models")?)], + )?; + let seed_locations = native( + py, + "build_seed_locations", + &[("seeds", project.getattr("seeds")?)], + )?; + let function_locations = native( + py, + "build_function_locations", + &[("functions", project.getattr("functions")?)], + )?; + let source_map = PyDict::new(py); + for source in source_project.getattr("sources")?.try_iter()? { + let source = source?; + source_map.set_item(source.getattr("name")?, source.getattr("source_entry")?)?; + } + let source_warehouse_columns = native( + py, + "gather_source_columns", + &[ + ("project", source_project.clone()), + ("adapter", adapter.clone()), + ("connection", connection.clone()), + ], + )?; + Ok(CloneResolution { + model_locations, + seed_locations, + function_locations, + source_map: source_map.into_any(), + source_warehouse_columns, + }) +} + +fn model_entry<'py>( + model: &Bound<'py, PyAny>, + materialization_type: Bound<'py, PyAny>, + resolved_sql: Bound<'py, PyAny>, +) -> PyResult> { + let py = model.py(); + let action = if materialization_type.eq(VIEW_MATERIALIZATION)? { + "CREATE_VIEW" + } else { + "CREATE_TABLE" + }; + planner_model( + py, + "ModelPlanEntry", + &[ + ("key", model.getattr("key")?), + ("name", model.getattr("name")?), + ("relative_path", model.getattr("relative_path")?), + ("materialization_type", materialization_type), + ("action", planner_type(py, "PlanAction", action)?), + ("reason", planner_type(py, "PlanReason", "NO_CHANGE")?), + ("destination", model.getattr("destination")?), + ("fingerprint_query_sql", string(py, "")), + ("resolved_sql", resolved_sql), + ("logical_ddl", string(py, "")), + ], + ) +} + +fn clone_model_entries<'py>( + (project, plan): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + source_project: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let resolution = clone_resolution(project, source_project, (adapter, connection))?; + let selected = plan.getattr("selected_keys")?; + let entries = PyDict::new(py); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + if !selected.contains(model.getattr("key")?)? || disabled_model(&model)? { + continue; + } + let materialization_type = + native(py, "get_materialization_type", &[("model", model.clone())])?; + let resolved_sql = if materialization_type.eq(VIEW_MATERIALIZATION)? { + let functions = PyDict::new(py); + for function in project.getattr("functions")?.try_iter()? { + let function = function?; + functions.set_item(function.getattr("name")?, function)?; + } + let context = planner_model( + py, + "ModelPlanContext", + &[ + ("model_locations", resolution.model_locations.clone()), + ("models_by_name", PyDict::new(py).into_any()), + ("functions_by_name", functions.into_any()), + ("seed_locations", resolution.seed_locations.clone()), + ("function_locations", resolution.function_locations.clone()), + ("source_map", resolution.source_map.clone()), + ( + "source_warehouse_columns", + resolution.source_warehouse_columns.clone(), + ), + ( + "star_exclude_keyword", + adapter.call_method0("star_exclude_keyword")?, + ), + ], + )?; + native( + py, + "resolve_model_sql", + &[ + ("adapter", adapter.clone()), + ("model", model.clone()), + ("snapshot", planner_model(py, "WarehouseSnapshot", &[])?), + ("context", context), + ( + "backfill", + planner_model( + py, + "BackfillResult", + &[( + "action", + planner_type(py, "BackfillAction", "FORWARD_ONLY")?, + )], + )?, + ), + ("full_refresh", PyBool::new(py, false).to_owned().into_any()), + ( + "cursor_overrides", + planner_model(py, "CursorOverridePair", &[])?, + ), + ], + )? + } else { + string(py, "") + }; + entries.set_item( + model.getattr("key")?, + model_entry(&model, materialization_type, resolved_sql)?, + )?; + } + in_order(&entries, &plan.getattr("execution_order")?) +} + +fn clone_source_entries<'py>( + project: &Bound<'py, PyAny>, + (selected, execution_order): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let entries = PyDict::new(py); + for source in project.getattr("sources")?.try_iter()? { + let source = source?; + let entry = source.getattr("source_entry")?; + if !selected.contains(source.getattr("key")?)? + || !entry.getattr("managed")?.is_truthy()? + || entry.getattr("loader")?.is_none() + || !entry.getattr("expression")?.is_none() + { + continue; + } + let table = entry.getattr("table")?; + let relation_name = if table.is_truthy()? { + table + } else { + entry.getattr("name")? + }; + let kwargs = PyDict::new(py); + kwargs.set_item("database", entry.getattr("database")?)?; + kwargs.set_item("schema", entry.getattr("schema")?)?; + kwargs.set_item("name", &relation_name)?; + let qualified = adapter + .getattr("render_qualified_name")? + .call((), Some(&kwargs))?; + let destination = imported(py, COMPILE_MODELS, "CompiledRelationLocation")?.call( + (), + Some(&{ + let location = PyDict::new(py); + location.set_item("database", entry.getattr("database")?)?; + location.set_item("schema", entry.getattr("schema")?)?; + location.set_item("name", relation_name)?; + location.set_item("qualified_name", qualified)?; + location + }), + )?; + entries.set_item( + source.getattr("key")?, + planner_model( + py, + "CloneSourcePlanEntry", + &[ + ("key", source.getattr("key")?), + ("name", source.getattr("name")?), + ("destination", destination), + ], + )?, + )?; + } + in_order(&entries, execution_order) +} + +fn clone_function_entries<'py>( + (project, plan): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + source_project: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let resolution = clone_resolution(project, source_project, (adapter, connection))?; + let selected = plan.getattr("selected_keys")?; + let entries = PyDict::new(py); + for function in project.getattr("functions")?.try_iter()? { + let function = function?; + if !selected.contains(function.getattr("key")?)? { + continue; + } + let body_sql = native( + py, + "resolve_function_sql", + &[ + ("adapter", adapter.clone()), + ("function", function.clone()), + ("model_locations", resolution.model_locations.clone()), + ("seed_locations", resolution.seed_locations.clone()), + ("function_locations", resolution.function_locations.clone()), + ("source_map", resolution.source_map.clone()), + ( + "source_warehouse_columns", + resolution.source_warehouse_columns.clone(), + ), + ( + "star_exclude_keyword", + adapter.call_method0("star_exclude_keyword")?, + ), + ], + )?; + let field = |name: &str| function.getattr(name); + let entry = planner_model( + py, + "FunctionPlanEntry", + &[ + ("key", field("key")?), + ("name", field("name")?), + ("relative_path", field("relative_path")?), + ("destination", field("destination")?), + ("arguments", field("arguments")?), + ("returns", field("returns")?), + ("body_sql", body_sql), + ( + "fingerprint_query_sql", + native( + py, + "build_compiled_function_fingerprint_sql", + &[("function", function.clone())], + )?, + ), + ("fingerprint_destination", field("fingerprint_destination")?), + ("return_columns", field("return_columns")?), + ("language", field("language")?), + ("source_file_path", field("source_file_path")?), + ("runtime_version", field("runtime_version")?), + ("entry_point", field("entry_point")?), + ("packages", field("packages")?), + ], + )?; + entries.set_item(function.getattr("key")?, entry)?; + } + in_order(&entries, &plan.getattr("execution_order")?) +} + +fn seed_entry<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + let py = seed.py(); + planner_model( + py, + "SeedPlanEntry", + &[ + ("key", seed.getattr("key")?), + ("name", seed.getattr("name")?), + ("destination", seed.getattr("destination")?), + ( + "file_path", + seed.getattr("seed_file")?.getattr("file_path")?, + ), + ("columns", PyTuple::empty(py).into_any()), + ( + "csv_settings", + seed.getattr("schema_entry")?.getattr("csv_settings")?, + ), + ], + ) +} + +/// Seeds whose `field` (the key or the name) is in `wanted`, as clone seed entries. +fn seed_entries<'py>( + project: &Bound<'py, PyAny>, + field: &str, + wanted: &Bound<'py, PyAny>, +) -> PyResult> { + let mut entries: Vec> = Vec::new(); + for seed in project.getattr("seeds")?.try_iter()? { + let seed = seed?; + if wanted.contains(seed.getattr(field)?)? { + entries.push(seed_entry(&seed)?); + } + } + tuple_of(project.py(), entries) +} + +fn origin_model_entries<'py>( + project: &Bound<'py, PyAny>, + selected_names: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let mut entries: Vec> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + if !selected_names.contains(model.getattr("name")?)? || disabled_model(&model)? { + continue; + } + let materialization_type = + native(py, "get_materialization_type", &[("model", model.clone())])?; + entries.push(model_entry(&model, materialization_type, string(py, ""))?); + } + tuple_of(py, entries) +} + +fn names_of<'py>(entries: &Bound<'py, PyAny>) -> PyResult> { + let mut names: Vec> = Vec::new(); + for entry in entries.try_iter()? { + names.push(entry?.getattr("name")?); + } + Ok(PyFrozenSet::new(entries.py(), names)?.into_any()) +} + +fn origin_source_entries<'py>( + project: &Bound<'py, PyAny>, + selected_names: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let mut keys: Vec> = Vec::new(); + let mut order: Vec> = Vec::new(); + for source in project.getattr("sources")?.try_iter()? { + let source = source?; + if selected_names.contains(source.getattr("name")?)? { + keys.push(source.getattr("key")?); + } + order.push(source.getattr("key")?); + } + clone_source_entries( + project, + ( + &PyFrozenSet::new(py, keys)?.into_any(), + &tuple_of(py, order)?, + ), + adapter, + ) +} + +/// Prepare planner outputs needed for clone execution. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn run_clone_planning<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "run_clone_planning", + kwargs, + ( + &[ + "project", + "select", + "exclude", + "adapter", + "connection", + "origin_project", + "origin_source_project", + "destination_source_project", + ], + &[], + ), + )?; + let project = args.value("project")?; + let adapter = args.value("adapter")?; + let connection = args.value("connection")?; + let destination_sources = args.value("destination_source_project")?; + let plan = clone_plan_output(&project, &args.value("select")?, &args.value("exclude")?)?; + let models = clone_model_entries( + (&project, &plan), + (&adapter, &connection), + &destination_sources, + )?; + let seeds = seed_entries(&project, "key", &plan.getattr("selected_keys")?)?; + let functions = clone_function_entries( + (&project, &plan), + (&adapter, &connection), + &destination_sources, + )?; + let mut sources = PyTuple::empty(py).into_any(); + if destination_sources + .getattr("effective_target_name")? + .eq(project.getattr("effective_target_name")?)? + { + sources = clone_source_entries( + &project, + ( + &plan.getattr("selected_keys")?, + &plan.getattr("execution_order")?, + ), + &adapter, + )?; + } + let origin_project = args.value("origin_project")?; + let origin_models = origin_model_entries(&origin_project, &names_of(&models)?)?; + let origin_seeds = seed_entries(&origin_project, "name", &names_of(&seeds)?)?; + let origin_sources = origin_source_entries( + &args.value("origin_source_project")?, + &names_of(&sources)?, + &adapter, + )?; + let origin_by_name = PyDict::new(py); + for entry in origin_sources.try_iter()? { + let entry = entry?; + origin_by_name.set_item(entry.getattr("name")?, entry)?; + } + let mut kept_sources: Vec> = Vec::new(); + let mut kept_origins: Vec> = Vec::new(); + for entry in sources.try_iter()? { + let entry = entry?; + let Some(origin) = origin_by_name.get_item(entry.getattr("name")?)? else { + continue; + }; + if origin + .getattr("destination")? + .ne(entry.getattr("destination")?)? + { + kept_origins.push(origin); + kept_sources.push(entry); + } + } + PyTuple::new( + py, + [ + plan, + tuple_of(py, kept_sources)?, + models, + seeds, + functions, + origin_models, + origin_seeds, + tuple_of(py, kept_origins)?, + ], + ) +} + +/// The traversal callbacks and graph of one deferred-clone query. +struct CloneGraph<'py> { + selected: Bound<'py, PyAny>, + upstream: Bound<'py, PyAny>, + is_clonable: Bound<'py, PyAny>, + is_view: Bound<'py, PyAny>, +} + +impl<'py> CloneGraph<'py> { + fn read(args: &KeywordArguments<'py>) -> PyResult { + Ok(Self { + selected: args.value("selected")?, + upstream: args.value("upstream")?, + is_clonable: args.value("is_clonable")?, + is_view: args.value("is_view")?, + }) + } + + fn parents(&self, node: &Bound<'py, PyAny>) -> PyResult>> { + self.upstream + .call_method1("get", (node, PyTuple::empty(node.py())))? + .try_iter()? + .collect() + } + + fn holds(&self, check: &Bound<'py, PyAny>, node: &Bound<'py, PyAny>) -> PyResult { + check.call1((node,))?.is_truthy() + } + + /// Depth-first walk from every selected node in Python's recursion order. + fn walk(&self, skipped_views: bool) -> PyResult> { + let py = self.selected.py(); + let visited = PySet::empty(py)?; + let found = PySet::empty(py)?; + for start in self.selected.try_iter()? { + let start = start?; + if visited.contains(&start)? { + continue; + } + visited.add(&start)?; + let mut stack: Vec<(Vec>, usize)> = vec![(self.parents(&start)?, 0)]; + while let Some((parents, index)) = stack.last_mut() { + let Some(parent) = parents.get(*index).cloned() else { + stack.pop(); + continue; + }; + *index += 1; + let descend = if self.selected.contains(&parent)? { + true + } else if skipped_views { + let view = self.holds(&self.is_view, &parent)?; + if view && self.holds(&self.is_clonable, &parent)? { + found.add(&parent)?; + } + view + } else if self.holds(&self.is_view, &parent)? { + true + } else { + if self.holds(&self.is_clonable, &parent)? { + found.add(&parent)?; + } + false + }; + if descend && !visited.contains(&parent)? { + visited.add(&parent)?; + stack.push((self.parents(&parent)?, 0)); + } + } + } + PyFrozenSet::new(py, found.iter()) + } +} + +/// Return non-view clonable ancestors anchoring selected nodes. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn resolve_clone_boundary<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "resolve_clone_boundary", + kwargs, + (&GRAPH_ARGUMENTS, &[]), + )?; + CloneGraph::read(&args)?.walk(false) +} + +/// Return skipped view ancestors rebuilt over cloned boundaries. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn resolve_skipped_view_chain<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "resolve_skipped_view_chain", + kwargs, + (&GRAPH_ARGUMENTS, &[]), + )?; + CloneGraph::read(&args)?.walk(true) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_clone_planning, module)?)?; + module.add_function(wrap_pyfunction!(resolve_clone_boundary, module)?)?; + module.add_function(wrap_pyfunction!(resolve_skipped_view_chain, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs new file mode 100644 index 0000000000..2086ff2412 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -0,0 +1,470 @@ +//! Static plans for commands that do not build models, and direct seed identity reads. + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::planner_entries::plan_output::{ + function_entry, selected_test_entries, test_selected, +}; +use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; +use crate::bindings::_helpers::planner_entries::values::{ + boolean, native, none, planner_model, planner_type, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use std::collections::{BTreeMap, BTreeSet}; + +const FINGERPRINT_CONSTANTS: &str = "sqlbuild.compiler.fingerprints.constants"; + +/// Optional fields of a static command plan; absent ones take `PlanOutput` defaults. +#[derive(Default)] +struct StaticPlanParts<'py> { + execution_order: Option>, + fields: Vec<(&'static str, Bound<'py, PyAny>)>, +} + +fn base_plan_output<'py>( + project: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, + parts: StaticPlanParts<'py>, +) -> PyResult> { + let py = project.py(); + let selected_keys = scope.getattr("selected_keys")?; + let execution_order = match parts.execution_order { + Some(order) => order, + None => { + let mut order: Vec> = Vec::new(); + for key in scope.getattr("execution_order")?.try_iter()? { + let key = key?; + if selected_keys.contains(&key)? { + order.push(key); + } + } + PyTuple::new(py, order)?.into_any() + } + }; + let mut fields = vec![("execution_order", execution_order)]; + fields.extend(parts.fields); + fields.extend([ + ("selected_keys", selected_keys), + ("upstream_deps", scope.getattr("upstream_deps")?), + ("downstream_deps", scope.getattr("downstream_deps")?), + ("model_locations", relations.getattr("model_locations")?), + ("seed_locations", relations.getattr("seed_locations")?), + ( + "function_locations", + relations.getattr("function_locations")?, + ), + ("source_map", relations.getattr("source_map")?), + ("source_read_map", relations.getattr("source_read_map")?), + ( + "python_source_read_map", + relations.getattr("python_source_read_map")?, + ), + ("hook_functions", project.getattr("hook_functions")?), + ( + "enforce_explicit_references", + project.getattr("enforce_explicit_references")?, + ), + ]); + planner_model(py, "PlanOutput", &fields) +} + +/// Resolve command selection through the canonical build selector phase. +#[pyfunction] +#[pyo3(signature = (*, project, selection, auto_load_sources = false))] +fn resolve_static_command_scope<'py>( + project: &Bound<'py, PyAny>, + selection: &Bound<'py, PyAny>, + auto_load_sources: bool, +) -> PyResult> { + let py = project.py(); + native( + py, + "resolve_planner_scopes", + &[ + ("project", project.clone()), + ("selection", selection.clone()), + ( + "policies", + planner_model( + py, + "PlannerPolicies", + &[("auto_load_sources", boolean(py, auto_load_sources)?)], + )?, + ), + ], + )? + .getattr("selected_scope") +} + +/// Resolve canonical locations and source routing without warehouse inventory. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn resolve_static_relation_context<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let options = kwargs.unwrap_or_else(|| PyDict::new(py)); + KeywordArguments::read( + py, + "resolve_static_relation_context", + Some(options.clone()), + ( + &["project", "adapter", "scope"], + &[ + "deferral", + "project_config", + "local_config", + "relation_keys", + ], + ), + )?; + options.set_item("connection", none(py))?; + options.set_item("known_source_columns", PyDict::new(py))?; + let args = KeywordArguments::read( + py, + "resolve_static_relation_context", + Some(options), + ( + &["project", "adapter", "scope", "connection"], + &[ + "deferral", + "project_config", + "local_config", + "relation_keys", + "known_source_columns", + ], + ), + )?; + planner_relations_context(&args) +} + +/// Project selected audits from canonical static state. +#[pyfunction] +#[pyo3(signature = (*, project, adapter, scope, relations))] +fn build_audit_command_plan<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let materializations = PyDict::new(py); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + materializations.set_item( + model.getattr("name")?, + native(py, "get_materialization_type", &[("model", model.clone())])? + .getattr("value")?, + )?; + } + let audits = native( + py, + "build_selected_audit_entries", + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("scope", scope.clone()), + ("relations", relations.clone()), + ("model_materializations", materializations.into_any()), + ], + )?; + base_plan_output( + project, + scope, + relations, + StaticPlanParts { + execution_order: None, + fields: vec![( + "audit_entries", + PyTuple::new(py, audits.try_iter()?.collect::>>()?)?.into_any(), + )], + }, + ) +} + +/// Project selected SQL tests from canonical static state. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn build_test_command_plan<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_test_command_plan", + kwargs, + ( + &["project", "adapter", "scope", "relations"], + &["case_name"], + ), + )?; + let project = args.value("project")?; + let adapter = args.value("adapter")?; + let scope = args.value("scope")?; + let relations = args.value("relations")?; + let selected_keys = scope.getattr("selected_keys")?; + let selection = scope.getattr("sql_test_selection")?; + let selected = selected_test_entries( + (&project, &adapter), + &selected_keys, + &selection, + &args.value("case_name")?, + )?; + let (entries, warnings) = (selected.entries, selected.warnings); + let mut case_names: BTreeSet = BTreeSet::new(); + for test in project.getattr("sql_tests")?.try_iter()? { + let test = test?; + let case_name = test.getattr("case_name")?; + if !case_name.is_none() && test_selected(&test, &selected_keys, &selection)? { + case_names.insert(case_name.extract()?); + } + } + let mut test_keys: Vec> = Vec::new(); + let mut function_keys: Vec> = Vec::new(); + for entry in &entries { + test_keys.push(entry.getattr("key")?); + for key in entry.getattr("function_deps")?.try_iter()? { + function_keys.push(key?); + } + } + let function_keys = PyFrozenSet::new(py, function_keys)?; + let mut functions: Vec> = Vec::new(); + for function in project.getattr("functions")?.try_iter()? { + let function = function?; + if !function_keys.contains(function.getattr("key")?)? { + continue; + } + let fingerprint_sql = native( + py, + "build_compiled_function_fingerprint_sql", + &[("function", function.clone())], + )?; + functions.push(function_entry( + &function, + &adapter, + &relations, + ( + fingerprint_sql, + none(py), + planner_type(py, "PlanReason", "NO_CHANGE")?, + ), + )?); + } + let scoped_keys = selected_keys.call_method1("__or__", (PyFrozenSet::new(py, test_keys)?,))?; + let mut order: Vec> = Vec::new(); + for key in scope.getattr("execution_order")?.try_iter()? { + let key = key?; + if scoped_keys.contains(&key)? { + order.push(key); + } + } + base_plan_output( + &project, + &scope, + &relations, + StaticPlanParts { + execution_order: Some(PyTuple::new(py, order)?.into_any()), + fields: vec![ + ("test_entries", PyTuple::new(py, entries)?.into_any()), + ( + "available_test_case_names", + PyTuple::new(py, case_names)?.into_any(), + ), + ("function_entries", PyTuple::new(py, functions)?.into_any()), + ("warnings", PyTuple::new(py, warnings)?.into_any()), + ], + }, + ) +} + +/// Project selected direct seed work and canonical identities. +#[pyfunction] +#[pyo3(signature = (*, project, scope, relations, fingerprints = None))] +fn build_seed_command_plan<'py>( + project: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, + fingerprints: Option>, +) -> PyResult> { + let py = project.py(); + let existing = match fingerprints { + Some(fingerprints) if fingerprints.is_truthy()? => fingerprints, + _ => PyDict::new(py).into_any(), + }; + let selected_keys = scope.getattr("selected_keys")?; + let mut entries: Vec> = Vec::new(); + for seed in project.getattr("seeds")?.try_iter()? { + let seed = seed?; + if !selected_keys.contains(seed.getattr("key")?)? { + continue; + } + let identity = native(py, "build_seed_identity", &[("seed", seed.clone())])?; + let (version_hash, metadata_json) = (identity.get_item(0)?, identity.get_item(1)?); + let previous = existing.call_method1("get", (seed.getattr("name")?,))?; + let reason = if previous.is_none() { + "FIRST_RUN" + } else if previous.getattr("version_hash")?.ne(&version_hash)? { + "CONFIG_CHANGED" + } else { + "NO_CHANGE" + }; + entries.push(planner_model( + py, + "SeedPlanEntry", + &[ + ("key", seed.getattr("key")?), + ("name", seed.getattr("name")?), + ("destination", seed.getattr("destination")?), + ( + "file_path", + seed.getattr("seed_file")?.getattr("file_path")?, + ), + ( + "columns", + native(py, "extract_seed_columns", &[("seed", seed.clone())])?, + ), + ( + "csv_settings", + seed.getattr("schema_entry")?.getattr("csv_settings")?, + ), + ("fingerprint_definition", metadata_json.clone()), + ("fingerprint_version_hash", version_hash), + ("fingerprint_metadata_json", metadata_json), + ("reason", planner_type(py, "PlanReason", reason)?), + ], + )?); + } + base_plan_output( + project, + scope, + relations, + StaticPlanParts { + execution_order: None, + fields: vec![("seed_entries", PyTuple::new(py, entries)?.into_any())], + }, + ) +} + +/// Project static relation data for Python checks. +#[pyfunction] +#[pyo3(signature = (*, project, scope, relations))] +fn build_relation_command_plan<'py>( + project: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, +) -> PyResult> { + base_plan_output(project, scope, relations, StaticPlanParts::default()) +} + +/// A schema target's sort key: the database with `None` first as an empty name, then the schema. +type Target = (Option, String); + +/// Read only fingerprint state required by selected direct seeds. +#[pyfunction] +#[pyo3(signature = (*, adapter, connection, seeds))] +fn read_selected_seed_fingerprints<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + seeds: &Bound<'py, PyAny>, +) -> PyResult> { + let py = adapter.py(); + let table_name: String = + imported(py, FINGERPRINT_CONSTANTS, "FINGERPRINT_TABLE_NAME")?.extract()?; + let seed_node_type = imported(py, FINGERPRINT_CONSTANTS, "NODE_TYPE_SEED")?; + let mut names_by_target: BTreeMap<(String, String), (Target, BTreeSet)> = + BTreeMap::new(); + for seed in seeds.try_iter()? { + let seed = seed?; + let destination = seed.getattr("destination")?; + let schema = destination.getattr("schema")?; + if schema.is_none() { + continue; + } + let database: Option = destination.getattr("database")?.extract()?; + let schema: String = schema.extract()?; + names_by_target + .entry((database.clone().unwrap_or_default(), schema.clone())) + .or_insert_with(|| ((database, schema), BTreeSet::new())) + .1 + .insert(seed.getattr("name")?.extract()?); + } + let fingerprints = PyDict::new(py); + let mut relations_by_target: Vec> = Vec::new(); + for ((database, schema), _) in names_by_target.values() { + let names = PyTuple::new(py, [table_name.as_str()])?; + let schemas = PyTuple::new(py, [schema.as_str()])?; + let kwargs = PyDict::new(py); + kwargs.set_item("connection", connection)?; + kwargs.set_item("database", database)?; + kwargs.set_item("schemas", schemas)?; + kwargs.set_item("names", names)?; + relations_by_target.push(adapter.getattr("list_relations")?.call((), Some(&kwargs))?); + } + for (((database, schema), names), relations) in + names_by_target.values().zip(relations_by_target) + { + let mut table_exists = false; + for relation in relations.try_iter()? { + let name: String = relation?.getattr("name")?.extract()?; + if name.to_lowercase() == table_name.to_lowercase() { + table_exists = true; + break; + } + } + let node_names = PyTuple::new(py, names.iter())?; + let latest = native( + py, + "read_latest_fingerprints", + &[ + ("connection", connection.clone()), + ("execute", adapter.getattr("execute")?), + ("table_exists", boolean(py, table_exists)?), + ( + "database", + database + .as_deref() + .map_or_else(|| none(py), |name| PyString::new(py, name).into_any()), + ), + ("schema", PyString::new(py, schema).into_any()), + ( + "render_qualified_name", + adapter.getattr("render_qualified_name")?, + ), + ( + "render_read_latest_sql", + adapter.getattr("render_read_latest_fingerprints_sql")?, + ), + ("node_names", node_names.into_any()), + ], + )?; + let items = PyList::new( + py, + latest + .getattr("fingerprints")? + .call_method0("items")? + .try_iter()? + .collect::>>()?, + )?; + for item in items.iter() { + let (name, fingerprint) = (item.get_item(0)?, item.get_item(1)?); + if fingerprint.getattr("node_type")?.eq(&seed_node_type)? + && names.contains(&name.extract::()?) + { + fingerprints.set_item(name, fingerprint)?; + } + } + } + Ok(fingerprints) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_static_command_scope, module)?)?; + module.add_function(wrap_pyfunction!(resolve_static_relation_context, module)?)?; + module.add_function(wrap_pyfunction!(build_audit_command_plan, module)?)?; + module.add_function(wrap_pyfunction!(build_test_command_plan, module)?)?; + module.add_function(wrap_pyfunction!(build_seed_command_plan, module)?)?; + module.add_function(wrap_pyfunction!(build_relation_command_plan, module)?)?; + module.add_function(wrap_pyfunction!(read_selected_seed_fingerprints, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs new file mode 100644 index 0000000000..ff15696e1c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs @@ -0,0 +1,169 @@ +//! Display-only plans for external plan surfaces when warehouse planning cannot resolve. + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::planner_entries::values::{ + NATIVE_MODULE, boolean, native, none, planner_model, planner_type, string, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyFrozenSet, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +const VIEW: &str = "view"; +const INCREMENTAL: &str = "incremental"; +const CUSTOM: &str = "custom"; + +fn display_action( + materialization_type: &Bound<'_, PyAny>, + full_refresh: bool, +) -> PyResult<&'static str> { + Ok(if materialization_type.eq(VIEW)? { + "CREATE_VIEW" + } else if materialization_type.eq(INCREMENTAL)? { + if full_refresh { + "CREATE_TABLE" + } else { + "INCREMENTAL_APPEND" + } + } else if materialization_type.eq(CUSTOM)? { + "CUSTOM" + } else { + "CREATE_TABLE" + }) +} + +fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { + native( + model.py(), + "get_config_str", + &[ + ("values", model.getattr("config")?.getattr("values")?), + ("key", string(model.py(), key)), + ], + ) +} + +fn display_entry<'py>( + model: &Bound<'py, PyAny>, + full_refresh: bool, +) -> PyResult> { + let py = model.py(); + let materialization_type = native(py, "get_materialization_type", &[("model", model.clone())])?; + let effective = native( + py, + "resolve_model_full_refresh", + &[ + ("model", model.clone()), + ("cli_full_refresh", boolean(py, full_refresh)?), + ], + )? + .is_truthy()?; + let query_sql = model.getattr("query_sql")?; + let custom_name = if materialization_type.eq(CUSTOM)? { + config(model, "materialized")? + } else { + none(py) + }; + planner_model( + py, + "ModelPlanEntry", + &[ + ("key", model.getattr("key")?), + ("name", model.getattr("name")?), + ("relative_path", model.getattr("relative_path")?), + ( + "action", + planner_type( + py, + "PlanAction", + display_action(&materialization_type, effective)?, + )?, + ), + ("materialization_type", materialization_type), + ( + "reason", + planner_type( + py, + "PlanReason", + if effective { + "FULL_REFRESH" + } else { + "NO_CHANGE" + }, + )?, + ), + ("destination", model.getattr("destination")?), + ("fingerprint_query_sql", query_sql.clone()), + ("resolved_sql", query_sql), + ("logical_ddl", string(py, "")), + ( + "incremental_strategy", + config(model, "incremental_strategy")?, + ), + ("incremental_mode", config(model, "incremental_mode")?), + ("cursor_column", config(model, "cursor_column")?), + ("cursor_type", config(model, "cursor_type")?), + ( + "backfill", + planner_model( + py, + "BackfillResult", + &[( + "action", + planner_type( + py, + "BackfillAction", + if effective { "FULL" } else { "FORWARD_ONLY" }, + )?, + )], + )?, + ), + ("custom_materialization_name", custom_name), + ], + ) +} + +/// Build a display-only plan when warehouse planning cannot fully resolve. +#[pyfunction] +#[pyo3(signature = (*, project, selected_model_names, full_refresh))] +fn build_display_only_sqlbuild_plan<'py>( + project: &Bound<'py, PyAny>, + selected_model_names: &Bound<'py, PyAny>, + full_refresh: bool, +) -> PyResult> { + let py: Python<'py> = project.py(); + let selected = PyFrozenSet::new( + py, + selected_model_names + .try_iter()? + .collect::>>()?, + )?; + let mut entries: Vec> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + if selected.contains(model.getattr("name")?)? { + entries.push(display_entry(&model, full_refresh)?); + } + } + let keys: Vec> = entries + .iter() + .map(|entry| entry.getattr("key")) + .collect::>()?; + let upstream = + imported(py, NATIVE_MODULE, "build_execution_upstream_deps")?.call1((project,))?; + let downstream = imported(py, NATIVE_MODULE, "build_downstream_deps")?.call1((&upstream,))?; + planner_model( + py, + "PlanOutput", + &[ + ("execution_order", PyTuple::new(py, &keys)?.into_any()), + ("model_entries", PyTuple::new(py, entries)?.into_any()), + ("selected_keys", PyFrozenSet::new(py, keys)?.into_any()), + ("upstream_deps", upstream), + ("downstream_deps", downstream), + ], + ) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_display_only_sqlbuild_plan, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs new file mode 100644 index 0000000000..a7b06a8daa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -0,0 +1,422 @@ +//! Top-level planner orchestration producing an execution plan. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, within, +}; +use crate::bindings::_helpers::planner_entries::values::{ + COMPILE_TYPES, boolean, native, planner_model, replaced, string, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use std::time::Instant; + +const INSPECTION_CATALOG: (&str, &str) = ( + "sqlbuild.adapter.relations.main.open_inspection_catalog", + "open_inspection_catalog", +); +const TEST_PLANNING_THREAD: &str = "sqlbuild-test-planning"; + +/// A future resolving to the planned SQL tests, planned on a daemon thread when enabled. +fn sql_test_planning<'py>( + (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (selected_keys, sql_test_selection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + enabled: bool, +) -> PyResult> { + let py = project.py(); + let future = imported(py, "concurrent.futures", "Future")?.call0()?; + if !enabled { + future.call_method1( + "set_result", + (planner_model( + py, + "PlannedSqlTests", + &[ + ("selected_keys", selected_keys.clone()), + ("sql_test_selection", sql_test_selection.clone()), + ], + )?,), + )?; + return Ok(future); + } + let kwargs = PyDict::new(py); + kwargs.set_item("future", &future)?; + kwargs.set_item("project", project)?; + kwargs.set_item("adapter", adapter)?; + kwargs.set_item("selected_keys", selected_keys)?; + kwargs.set_item("sql_test_selection", sql_test_selection)?; + call_with( + &imported(py, "threading", "Thread")?, + &[ + ( + "target", + imported(py, "sqlbuild._native", "plan_selected_sql_tests_into")?, + ), + ("kwargs", kwargs.into_any()), + ("name", string(py, TEST_PLANNING_THREAD)), + ("daemon", boolean(py, true)?), + ], + )? + .call_method0("start")?; + Ok(future) +} + +/// Selected scopes, and the overrides with cursor overrides typed for the selected models. +fn scopes_and_cursor_overrides<'py>( + args: &KeywordArguments<'py>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let project = args.value("project")?; + let py = project.py(); + let scopes = native( + py, + "resolve_planner_scopes", + &[ + ("project", project.clone()), + ("selection", args.value("selection")?), + ("policies", args.value("policies")?), + ], + )?; + let selected = scopes.getattr("selected_scope")?; + let models_by_name = selected.getattr("models_by_name")?; + let model_type = imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("MODEL")?; + let mut models: Vec> = Vec::new(); + for key in selected.getattr("selected_keys")?.try_iter()? { + let key = key?; + let name = key.getattr("name")?; + if key.getattr("resource_type")?.eq(&model_type)? && models_by_name.contains(&name)? { + models.push(models_by_name.get_item(&name)?); + } + } + let overrides = args.value("overrides")?; + let typed = native( + py, + "type_cursor_overrides", + &[ + ("cursor_overrides", overrides.getattr("cursor_overrides")?), + ("selected_models", PyTuple::new(py, models)?.into_any()), + ], + )?; + Ok(( + scopes, + replaced(&overrides, &[("cursor_overrides", typed)])?, + )) +} + +/// Execution and stale-warning change results for the inspected scopes. +fn change_results<'py>( + project: &Bound<'py, PyAny>, + (scopes, snapshot, identities): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + overrides: &Bound<'py, PyAny>, + policies: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py = project.py(); + let versions = identities.getattr("version_identities")?; + let changes = native( + py, + "detect_changes", + &[ + ("project", project.clone()), + ("scope", scopes.getattr("inspection_scope")?), + ("snapshot", snapshot.clone()), + ("full_refresh", overrides.getattr("full_refresh")?), + ( + "expected_version_hashes", + versions.getattr("model_version_hashes")?, + ), + ( + "expected_metadata_jsons", + versions.getattr("model_metadata_jsons")?, + ), + ], + )?; + if !policies.getattr("selection_diagnostics")?.is_truthy()? { + let empty = planner_model( + py, + "PlannerChangeResults", + &[ + ("models", PyDict::new(py).into_any()), + ("functions", PyDict::new(py).into_any()), + ], + )?; + return Ok((changes, empty)); + } + let stale = native( + py, + "detect_stale_warning_changes", + &[ + ("project", project.clone()), + ("scopes", scopes.clone()), + ("snapshot", snapshot.clone()), + ("identities", identities.clone()), + ("execution_changes", changes.clone()), + ], + )?; + Ok((changes, stale)) +} + +/// The plan phases after warehouse inspection, run inside the inspection catalog. +fn planned_output<'py>( + args: &KeywordArguments<'py>, + runtime: &Bound<'py, PyAny>, + (scopes, overrides): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + test_planning: &Bound<'py, PyAny>, +) -> PyResult> { + let project = args.value("project")?; + let py = project.py(); + let deferral = args.value("deferral")?; + let policies = args.value("policies")?; + let warehouse = native( + py, + "gather_planner_warehouse_state", + &[ + ("runtime", runtime.clone()), + ("scopes", scopes.clone()), + ("overrides", overrides.clone()), + ("deferral", deferral.clone()), + ], + )?; + let started = Instant::now(); + let identities = native( + py, + "build_planner_identity_context", + &[ + ("project", project.clone()), + ("scopes", scopes.clone()), + ( + "include_stale_warning_identities", + policies.getattr("selection_diagnostics")?, + ), + ], + )?; + native( + py, + "check_selected_scope_buildability", + &[ + ("runtime", runtime.clone()), + ("scopes", scopes.clone()), + ("warehouse", warehouse.clone()), + ("deferral", deferral.clone()), + ], + )?; + let snapshot = warehouse.getattr("snapshot")?; + let (changes, stale_warning_changes) = change_results( + &project, + (scopes, &snapshot, &identities), + overrides, + &policies, + )?; + let inspection_scope = scopes.getattr("inspection_scope")?; + let resolved_actions = native( + py, + "resolve_model_actions", + &[ + ("scope", inspection_scope.clone()), + ("changes", changes.clone()), + ], + )?; + let source_freshness = native( + py, + "build_planner_source_freshness_result", + &[ + ("project", project.clone()), + ("adapter", args.value("adapter")?), + ("connection", args.value("connection")?), + ("scope", inspection_scope), + ("relations", warehouse.getattr("inspection_relations")?), + ( + "freshness_state_schemas", + snapshot.getattr("source_freshness_state_schemas")?, + ), + ], + )?; + let pruning = native( + py, + "prune_planner_execution_scope", + &[ + ("scopes", scopes.clone()), + ("resolved_actions", resolved_actions), + ], + )?; + let reconciliation = native( + py, + "reconcile_execution_changes", + &[ + ("warehouse", warehouse.clone()), + ("identities", identities.clone()), + ("pruning", pruning.clone()), + ("changes", changes), + ], + )?; + let entries = native( + py, + "build_planner_entry_results", + &[ + ("runtime", runtime.clone()), + ("warehouse", warehouse.clone()), + ("identities", identities.clone()), + ("overrides", overrides.clone()), + ("policies", policies.clone()), + ("deferral", deferral), + ("pruning", pruning.clone()), + ("reconciliation", reconciliation.clone()), + ("source_freshness", source_freshness.clone()), + ], + )?; + let planned_sql_tests = test_planning.call_method0("result")?; + let mut plan_output = native( + py, + "assemble_base_plan_output", + &[ + ("runtime", runtime.clone()), + ("warehouse", warehouse.clone()), + ("identities", identities.clone()), + ("overrides", overrides.clone()), + ("pruning", pruning.clone()), + ("reconciliation", reconciliation), + ("entries", entries), + ("source_freshness", source_freshness.clone()), + ("planned_sql_tests", planned_sql_tests), + ], + )?; + plan_output = native( + py, + "with_storage_policies", + &[ + ("plan_output", plan_output), + ("runtime", runtime.clone()), + ("warehouse", warehouse.clone()), + ("scopes", scopes.clone()), + ("policies", policies.clone()), + ], + )?; + plan_output = native( + py, + "with_plan_warnings", + &[ + ("runtime", runtime.clone()), + ("scopes", scopes.clone()), + ("warehouse", warehouse), + ("identities", identities), + ("stale_warning_changes", stale_warning_changes), + ("pruning", pruning.clone()), + ("source_freshness", source_freshness.clone()), + ("plan_output", plan_output), + ("policies", policies.clone()), + ], + )?; + plan_output = native( + py, + "with_plan_metadata", + &[ + ("plan_output", plan_output), + ("pruning", pruning), + ("source_freshness", source_freshness), + ("policies", policies), + ], + )?; + let on_progress = args.value("on_progress")?; + if !on_progress.is_none() { + on_progress.call1((format!( + "Generated plan. ({:.2}s)", + started.elapsed().as_secs_f64() + ),))?; + } + Ok(plan_output) +} + +/// Build the execution plan for one selection against live warehouse state. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn build_execution_plan<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_execution_plan", + kwargs, + ( + &[ + "project", + "adapter", + "connection", + "selection", + "overrides", + "deferral", + "policies", + ], + &["on_progress", "project_config", "local_config"], + ), + )?; + let project = args.value("project")?; + let adapter = args.value("adapter")?; + let connection = args.value("connection")?; + let runtime = planner_model( + py, + "PlannerRuntime", + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("connection", connection.clone()), + ("project_config", args.value("project_config")?), + ("local_config", args.value("local_config")?), + ("on_progress", args.value("on_progress")?), + ], + )?; + let (scopes, overrides) = scopes_and_cursor_overrides(&args)?; + let inspection = scopes.getattr("inspection_scope")?; + let test_planning = sql_test_planning( + (&project, &adapter), + ( + &inspection.getattr("selected_keys")?, + &inspection.getattr("sql_test_selection")?, + ), + args.value("policies")? + .getattr("plan_sql_tests")? + .is_truthy()?, + )?; + let catalog = call_with( + &imported(py, INSPECTION_CATALOG.0, INSPECTION_CATALOG.1)?, + &[("adapter", adapter), ("connection", connection)], + )?; + within(&catalog, |_| { + planned_output(&args, &runtime, (&scopes, &overrides), &test_planning) + }) +} + +/// Start SQL test planning on a daemon thread, or resolve to no tests when disabled. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn start_sql_test_planning<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "start_sql_test_planning", + kwargs, + ( + &[ + "project", + "adapter", + "selected_keys", + "sql_test_selection", + "enabled", + ], + &[], + ), + )?; + sql_test_planning( + (&args.value("project")?, &args.value("adapter")?), + ( + &args.value("selected_keys")?, + &args.value("sql_test_selection")?, + ), + args.flag("enabled")?, + ) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_execution_plan, module)?)?; + module.add_function(wrap_pyfunction!(start_sql_test_planning, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs new file mode 100644 index 0000000000..452f3c67a2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs @@ -0,0 +1,11 @@ +//! Plan entry and plan output assembly over native planner resolution. + +mod clone_plans; +mod command_plans; +mod display_plan; +mod execution_plan; +mod model_entries; +mod plan_output; +pub(crate) mod registration; +mod relations; +mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs new file mode 100644 index 0000000000..82e93fa088 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -0,0 +1,705 @@ +//! Per-model plan entries from resolved change results and warehouse snapshot facts. + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::planner_entries::values::{ + COMPILE_EXCEPTIONS, COMPILE_TYPES, PLANNER_CONSTANTS, PLANNER_EXCEPTIONS, PLANNER_TYPES, + boolean, error_of, native, none, planner_model, planner_type, replaced, string, +}; +use crate::bindings::_helpers::planner_resolution::cursor_ranges::{ + CursorWindowInputs, config_str, cursor_safety_configs, empty_cursor_inputs, microbatch_range, + model_cursor_overrides, plan_cursor_bounds, +}; +use crate::bindings::_helpers::planner_resolution::cursor_warnings::with_cursor_safety_warnings; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +const MICROBATCH_COMPLEXITY_CODE: &str = "MICROBATCH_CONCURRENCY_COMPLEXITY"; + +/// One model's planning request after change detection. +struct ModelPlanRequest<'py> { + model: Bound<'py, PyAny>, + snapshot: Bound<'py, PyAny>, + adapter: Bound<'py, PyAny>, + context: Bound<'py, PyAny>, + sql_analysis_enabled: Bound<'py, PyAny>, + full_refresh: bool, + overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), + change_result: Bound<'py, PyAny>, + external_resolver: Bound<'py, PyAny>, + protect_full_refresh: bool, +} + +/// Build a model plan entry and its warnings from a resolved change result. +fn plan_model_from_change<'py>( + request: &ModelPlanRequest<'py>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let model = &request.model; + let context = &request.context; + let py = model.py(); + let full_refresh = boolean(py, request.full_refresh)?; + let backfill = if request.full_refresh { + planner_model( + py, + "BackfillResult", + &[("action", planner_type(py, "BackfillAction", "FULL")?)], + )? + } else { + request.change_result.getattr("backfill")? + }; + let cursor_column = config_str(model, "cursor")?; + native( + py, + "validate_source_cursor_input_columns", + &[ + ("model", model.clone()), + ("cursor_column", cursor_column.clone()), + ("models_by_name", context.getattr("models_by_name")?), + ("functions_by_name", context.getattr("functions_by_name")?), + ("source_map", context.getattr("source_map")?), + ( + "source_warehouse_columns", + context.getattr("source_warehouse_columns")?, + ), + ], + )?; + let resolved = native( + py, + "resolve_model_plan_action", + &[ + ("model", model.clone()), + ("change_result", request.change_result.clone()), + ("full_refresh", full_refresh.clone()), + ], + )?; + let (action, reason) = (resolved.get_item(0)?, resolved.get_item(1)?); + let materialization_type = native(py, "get_materialization_type", &[("model", model.clone())])?; + let replaces_relation = native( + py, + "replaces_incremental_relation", + &[ + ("materialization_type", materialization_type.clone()), + ("action", action.clone()), + ], + )? + .is_truthy()?; + let full_rebuild_cause = if replaces_relation { + native( + py, + "describe_full_rebuild_cause", + &[ + ("change_result", request.change_result.clone()), + ("full_refresh", full_refresh.clone()), + ], + )? + } else { + none(py) + }; + if !full_rebuild_cause.is_none() && request.protect_full_refresh && !request.full_refresh { + native( + py, + "check_full_rebuild_allowed", + &[ + ("model", model.clone()), + ("change_result", request.change_result.clone()), + ("full_rebuild_cause", full_rebuild_cause.clone()), + ], + )?; + } + let pair = planner_model( + py, + "CursorOverridePair", + &[ + ("start_cursor_override", request.overrides.0.clone()), + ("end_cursor_override", request.overrides.1.clone()), + ], + )?; + let resolved_sql = native( + py, + "resolve_model_sql", + &[ + ("adapter", request.adapter.clone()), + ("model", model.clone()), + ("snapshot", request.snapshot.clone()), + ("context", context.clone()), + ("backfill", backfill.clone()), + ("full_refresh", full_refresh.clone()), + ("cursor_overrides", pair), + ( + "external_sql_reference_resolver", + request.external_resolver.clone(), + ), + ("replaces_relation", boolean(py, replaces_relation)?), + ("full_rebuild_cause", full_rebuild_cause), + ], + )?; + let hooks = native( + py, + "resolve_model_hook_entries", + &[ + ("model", model.clone()), + ("adapter", request.adapter.clone()), + ("context", context.clone()), + ( + "external_sql_reference_resolver", + request.external_resolver.clone(), + ), + ], + )?; + let (future_config, start_config) = cursor_safety_configs(model, context)?; + native( + py, + "validate_capped_watermark_inputs", + &[ + ("model", model.clone()), + ("models_by_name", context.getattr("models_by_name")?), + ("functions_by_name", context.getattr("functions_by_name")?), + ], + )?; + let cursor_input_relations = native( + py, + "build_cursor_input_relations", + &[ + ("model", model.clone()), + ("adapter", request.adapter.clone()), + ("model_locations", context.getattr("model_locations")?), + ("models_by_name", context.getattr("models_by_name")?), + ("functions_by_name", context.getattr("functions_by_name")?), + ("seed_locations", context.getattr("seed_locations")?), + ("source_map", context.getattr("source_map")?), + ("cursor_column", cursor_column.clone()), + ( + "runtime_cursor_producer_names", + context.getattr("runtime_cursor_producer_names")?, + ), + ], + )?; + let mut any_runtime_owned = false; + for relation in cursor_input_relations.try_iter()? { + if relation?.getattr("is_runtime_owned")?.is_truthy()? { + any_runtime_owned = true; + break; + } + } + let complete_override = !request.overrides.0.is_none() && !request.overrides.1.is_none(); + let runtime_owned = any_runtime_owned && !complete_override; + let window = CursorWindowInputs { + snapshot: request.snapshot.clone(), + backfill: backfill.clone(), + overrides: request.overrides.clone(), + runtime_owned, + policies: (future_config.clone(), start_config.clone()), + invocation_time: context.getattr("invocation_time")?, + full_refresh: request.full_refresh, + replaces_relation, + cursor_input_relations: cursor_input_relations.clone(), + }; + let cursor_bounds = plan_cursor_bounds(model, &window)?; + let (empty_inputs, waiting_on_empty_inputs) = empty_cursor_inputs(model, &window)?; + let range = microbatch_range(model, &window)?; + let assembled = native( + py, + "assemble_model_plan_entry", + &[ + ("model", model.clone()), + ("adapter", request.adapter.clone()), + ("snapshot", request.snapshot.clone()), + ("change_result", request.change_result.clone()), + ("action", action), + ("reason", reason), + ("materialization_type", materialization_type), + ("resolved_sql", resolved_sql), + ("sql_analysis_enabled", request.sql_analysis_enabled.clone()), + ("pre_hooks", hooks.get_item(0)?), + ("post_hooks", hooks.get_item(1)?), + ("cursor_column", cursor_column), + ("cursor_input_relations", cursor_input_relations), + ("runtime_owned_cursor_bounds", boolean(py, runtime_owned)?), + ("cursor_bounds", cursor_bounds.clone()), + ("microbatch_range", range.clone()), + ("empty_cursor_inputs", empty_inputs), + ( + "waiting_on_empty_inputs", + boolean(py, waiting_on_empty_inputs)?, + ), + ("backfill", backfill), + ("start_cursor_override", request.overrides.0.clone()), + ("end_cursor_override", request.overrides.1.clone()), + ("future_cursor_config", future_config), + ("start_cursor_config", start_config), + ("invocation_time", context.getattr("invocation_time")?), + ], + )?; + let bounds = if range.is_truthy()? { + range + } else { + cursor_bounds + }; + let warnings = + with_cursor_safety_warnings(&assembled.get_item(1)?, &model.getattr("name")?, &bounds)?; + Ok((assembled.get_item(0)?, warnings)) +} + +/// The binding-diagnostic failure that stops planning, if any diagnostic is an error. +fn check_binding_diagnostics<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py = project.py(); + let sql_test = imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SQL_TEST")?; + let mut errors: Vec> = Vec::new(); + for diagnostic in project.getattr("diagnostics")?.try_iter()? { + let diagnostic = diagnostic?; + if diagnostic.getattr("is_error")?.is_truthy()? + && diagnostic.getattr("resource_type")?.ne(&sql_test)? + { + errors.push(diagnostic); + } + } + let bound = native( + py, + "source_binding_diagnostics", + &[ + ("project", project.clone()), + ("columns", relations.getattr("source_warehouse_columns")?), + ( + "profile", + adapter.call_method0("expression_inference_profile")?, + ), + ("selected_keys", selected_keys.clone()), + ], + )?; + for diagnostic in bound.try_iter()? { + let diagnostic = diagnostic?; + if diagnostic.getattr("is_error")?.is_truthy()? { + errors.push(diagnostic); + } + } + let Some(first) = errors.first() else { + return Ok(()); + }; + let mut lines: Vec = Vec::new(); + for diagnostic in &errors { + let position = |field: &str| -> PyResult { + let value = diagnostic.getattr(field)?; + Ok(if value.is_truthy()? { + value.str()?.to_string() + } else { + "1".to_owned() + }) + }; + lines.push(format!( + "error[{}]: {}\n --> {}:{}:{}", + diagnostic.getattr("code")?.str()?, + diagnostic.getattr("message")?.str()?, + diagnostic.getattr("path")?.str()?, + position("line")?, + position("column")?, + )); + } + Err(error_of( + py, + (PLANNER_EXCEPTIONS, "PlannerInputError"), + &lines.join("\n"), + &[("code", first.getattr("code")?)], + )) +} + +fn names_of(entries: &[Bound<'_, PyAny>]) -> PyResult { + Ok(entries + .iter() + .map(|entry| Ok(entry.getattr("name")?.str()?.to_string())) + .collect::>>()? + .join(", ")) +} + +/// Concurrent microbatch configuration checks, and the warning concurrency complexity earns. +fn check_microbatch_concurrency<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + entries: &[Bound<'py, PyAny>], +) -> PyResult>> { + let py = project.py(); + let skip = planner_type(py, "PlanAction", "SKIP")?; + let create = planner_type(py, "PlanAction", "CREATE_TABLE")?; + let mut configured: Vec> = Vec::new(); + for entry in entries { + if entry.getattr("batch_concurrency")?.gt(1)? && entry.getattr("action")?.ne(&skip)? { + configured.push(entry.clone()); + } + } + let settings = project.getattr("settings")?; + if !configured.is_empty() && !settings.getattr("microbatch_concurrency")?.is_truthy()? { + let guidance = native(py, "microbatch_guidance", &[])?; + let prefix: String = + imported(py, PLANNER_CONSTANTS, "MICROBATCH_CONCURRENCY_MESSAGE")?.extract()?; + return Err(error_of( + py, + (COMPILE_EXCEPTIONS, "CompileInputError"), + &format!( + "{prefix}{}; {}", + names_of(&configured)?, + guidance.get_item(0)?.str()? + ), + &[("help", guidance.get_item(1)?)], + )); + } + if !configured.is_empty() + && !adapter + .call_method0("supports_concurrent_microbatch_dml")? + .is_truthy()? + { + return Err(error_of( + py, + (COMPILE_EXCEPTIONS, "CompileInputError"), + &format!( + "adapter '{}' does not support concurrent microbatch delete/insert DML; set \ + batch_concurrency = 1", + adapter.getattr("adapter_name")?.str()? + ), + &[], + )); + } + let mut concurrent: Vec> = Vec::new(); + for entry in configured { + if entry.getattr("action")?.ne(&create)? { + concurrent.push(entry); + } + } + if concurrent.is_empty() || !settings.getattr("concurrency")?.gt(1)? { + return Ok(None); + } + let count = concurrent.len(); + planner_model( + py, + "PlanWarning", + &[ + ("model_name", none(py)), + ("severity", planner_type(py, "WarningSeverity", "WARNING")?), + ("code", string(py, MICROBATCH_COMPLEXITY_CODE)), + ( + "message", + string( + py, + &format!( + "Concurrent microbatch execution is enabled for {count} selected \ + model{}: {}. Concurrent batches may complete out of order and rely \ + on persistent microbatch history for gap recovery and fingerprint \ + provenance. State deletion or corruption can require synthetic \ + reconciliation and weaken version guarantees. Keep \ + batch_concurrency at 1 unless the runtime benefit justifies this \ + complexity; see the microbatch concurrency and recovery \ + documentation.", + if count == 1 { "" } else { "s" }, + names_of(&concurrent)? + ), + ), + ), + ], + ) + .map(Some) +} + +/// Build model plan entries from snapshot and per-model resolved actions. +fn plan_entries<'py>(args: &KeywordArguments<'py>) -> PyResult> { + let project = args.value("project")?; + let py = project.py(); + let adapter = args.value("adapter")?; + let scope = args.value("scope")?; + let snapshot = args.value("snapshot")?; + let relations = args.value("relations")?; + let models = args.value("resolved_actions")?.getattr("models")?; + let cursor_overrides = args.value("cursor_overrides")?; + let full_refresh = args.value("full_refresh")?; + let inputs = args.value("build_inputs")?; + let inputs = if inputs.is_none() { + planner_model(py, "PlanEntryBuildInputs", &[])? + } else { + inputs + }; + let selected_keys = scope.getattr("selected_keys")?; + check_binding_diagnostics(&project, &adapter, &relations, &selected_keys)?; + let freshness_blocked = inputs.getattr("source_freshness_blocked_model_names")?; + let external_blocked = inputs.getattr("external_blocked_model_names")?; + let start_override = inputs.getattr("start_cursor_override")?; + let end_override = inputs.getattr("end_cursor_override")?; + let producers = native( + py, + "runtime_cursor_producer_names", + &[ + ("scope", scope.clone()), + ("source_map", relations.getattr("source_map")?), + ], + )?; + let functions_by_name = PyDict::new(py); + for function in project.getattr("functions")?.try_iter()? { + let function = function?; + functions_by_name.set_item(function.getattr("name")?, function)?; + } + let settings = project.getattr("settings")?; + let models_by_name = scope.getattr("models_by_name")?; + let context = planner_model( + py, + "ModelPlanContext", + &[ + ("model_locations", relations.getattr("model_locations")?), + ("models_by_name", models_by_name.clone()), + ("functions_by_name", functions_by_name.into_any()), + ("seed_locations", relations.getattr("seed_locations")?), + ( + "function_locations", + relations.getattr("function_locations")?, + ), + ("source_map", relations.getattr("source_read_map")?), + ( + "source_warehouse_columns", + relations.getattr("source_warehouse_columns")?, + ), + ( + "star_exclude_keyword", + relations.getattr("star_exclude_keyword")?, + ), + ("runtime_cursor_producer_names", producers), + ( + "future_cursor_config", + inputs.getattr("future_cursor_config")?, + ), + ( + "start_cursor_config", + inputs.getattr("start_cursor_config")?, + ), + ("invocation_time", inputs.getattr("invocation_time")?), + ], + )?; + let skip = planner_type(py, "PlanAction", "SKIP")?; + let microbatch = imported(py, PLANNER_TYPES, "IncrementalMode")?.getattr("MICROBATCH")?; + let mut entries: Vec> = Vec::new(); + let mut warnings: Vec> = Vec::new(); + for key in scope.getattr("execution_order")?.try_iter()? { + let key = key?; + let name = key.getattr("name")?; + if !selected_keys.contains(&key)? || !models.contains(&name)? { + continue; + } + let model = models_by_name.call_method1("get", (&name,))?; + let resolved = models.get_item(&name)?; + if model.is_none() { + continue; + } + let overrides = model_cursor_overrides( + &model, + &cursor_overrides, + (start_override.clone(), end_override.clone()), + )?; + let effective_full_refresh = native( + py, + "resolve_model_full_refresh", + &[ + ("model", model.clone()), + ("cli_full_refresh", full_refresh.clone()), + ], + )? + .is_truthy()?; + let (mut entry, entry_warnings) = plan_model_from_change(&ModelPlanRequest { + model: model.clone(), + snapshot: snapshot.clone(), + adapter: adapter.clone(), + context: context.clone(), + sql_analysis_enabled: settings.getattr("sql_analysis")?, + full_refresh: effective_full_refresh, + overrides, + change_result: resolved.getattr("change")?, + external_resolver: project.getattr("external_sql_reference_resolver")?, + protect_full_refresh: !freshness_blocked.contains(&name)? + && !external_blocked.contains(&name)?, + })?; + let entry_name = entry.getattr("name")?; + if freshness_blocked.contains(&entry_name)? { + entry = replaced( + &entry, + &[ + ("action", skip.clone()), + ( + "reason", + planner_type(py, "PlanReason", "SOURCE_FRESHNESS_ERROR")?, + ), + ], + )?; + } + if external_blocked.contains(&entry_name)? { + entry = replaced( + &entry, + &[ + ("action", skip.clone()), + ( + "reason", + planner_type(py, "PlanReason", "EXTERNAL_UPSTREAM_FAILED")?, + ), + ], + )?; + } + if entry.getattr("incremental_mode")?.eq(µbatch)? { + let policy = entry.getattr("unaccounted_partition_policy")?; + let policy = if policy.is_truthy()? { + policy + } else { + settings.getattr("microbatch_unaccounted_partition_policy")? + }; + entry = replaced(&entry, &[("unaccounted_partition_policy", policy)])?; + if entry.getattr("action")?.ne(&skip)? { + let limited = native( + py, + "apply_configured_microbatch_limits", + &[("entry", entry), ("inputs", inputs.clone())], + )?; + entry = limited.get_item(0)?; + for warning in limited.get_item(1)?.try_iter()? { + warnings.push(warning?); + } + } + } + entries.push(entry); + for warning in entry_warnings.try_iter()? { + warnings.push(warning?); + } + } + if let Some(warning) = check_microbatch_concurrency(&project, &adapter, &entries)? { + warnings.push(warning); + } + planner_model( + py, + "PlannerModelEntryResults", + &[ + ("entries", PyTuple::new(py, entries)?.into_any()), + ("warnings", PyTuple::new(py, warnings)?.into_any()), + ], + ) +} + +/// Build model plan entries from snapshot and per-model resolved actions. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn build_plan_entries<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_plan_entries", + kwargs, + ( + &[ + "project", + "adapter", + "scope", + "snapshot", + "relations", + "resolved_actions", + "cursor_overrides", + "full_refresh", + ], + &["build_inputs"], + ), + )?; + plan_entries(&args) +} + +/// Detect changes and build a plan entry and warnings for a single model. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn plan_model<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let args = KeywordArguments::read( + py, + "plan_model", + kwargs, + ( + &[ + "model", + "snapshot", + "adapter", + "context", + "sql_analysis_enabled", + "query_change_tracking", + "full_refresh", + "cursor_overrides", + ], + &["external_sql_reference_resolver"], + ), + )?; + let model = args.value("model")?; + let snapshot = args.value("snapshot")?; + let change_result = native( + py, + "detect_model_changes", + &[ + ("model", model.clone()), + ("snapshot", snapshot.clone()), + ("sql_analysis_enabled", args.value("sql_analysis_enabled")?), + ( + "query_change_tracking", + args.value("query_change_tracking")?, + ), + ("full_refresh", args.value("full_refresh")?), + ], + )?; + let overrides = args.value("cursor_overrides")?; + plan_model_from_change(&ModelPlanRequest { + model, + snapshot, + adapter: args.value("adapter")?, + context: args.value("context")?, + sql_analysis_enabled: args.value("sql_analysis_enabled")?, + full_refresh: args.flag("full_refresh")?, + overrides: ( + overrides.getattr("start_cursor_override")?, + overrides.getattr("end_cursor_override")?, + ), + change_result, + external_resolver: args.value("external_sql_reference_resolver")?, + protect_full_refresh: true, + }) +} + +/// Build a name-to-materialization-type lookup from planned model entries. +#[pyfunction] +#[pyo3(signature = (model_entries, /))] +fn build_model_materializations<'py>( + model_entries: &Bound<'py, PyAny>, +) -> PyResult> { + let lookup = PyDict::new(model_entries.py()); + for entry in model_entries.try_iter()? { + let entry = entry?; + lookup.set_item( + entry.getattr("name")?, + entry.getattr("materialization_type")?, + )?; + } + Ok(lookup) +} + +/// Check if any scope dependency is in the selected keys. +#[pyfunction] +#[pyo3(signature = (*, scope_deps, selected_keys))] +pub(crate) fn scope_overlaps( + scope_deps: &Bound<'_, PyAny>, + selected_keys: &Bound<'_, PyAny>, +) -> PyResult { + for dependency in scope_deps.try_iter()? { + if selected_keys.contains(dependency?)? { + return Ok(true); + } + } + Ok(false) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_plan_entries, module)?)?; + module.add_function(wrap_pyfunction!(plan_model, module)?)?; + module.add_function(wrap_pyfunction!(build_model_materializations, module)?)?; + module.add_function(wrap_pyfunction!(scope_overlaps, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs new file mode 100644 index 0000000000..c0d6f9153b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs @@ -0,0 +1,865 @@ +//! `PlanOutput` assembly from planned model entries, seeds, sources, functions, audits and tests. + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::planner_entries::model_entries::scope_overlaps; +use crate::bindings::_helpers::planner_entries::values::{ + DISCOVERY_MODELS, PLANNER_EXCEPTIONS, error_of, native, none, planner_model, planner_type, + string, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +const LOADER_CONSUMER: &str = "loader"; +const HOOK_CONSUMER: &str = "hook"; +const MATERIALIZATION_CONSUMER: &str = "custom materialization"; + +fn tuple_of<'py>(py: Python<'py>, items: Vec>) -> PyResult> { + Ok(PyTuple::new(py, items)?.into_any()) +} + +/// `mapping.get(key)` for an optional mapping. +fn optional_get<'py>( + mapping: &Bound<'py, PyAny>, + key: &Bound<'py, PyAny>, +) -> PyResult> { + if mapping.is_truthy()? { + return mapping.call_method1("get", (key,)); + } + Ok(none(mapping.py())) +} + +fn optional_get_or<'py>( + mapping: &Bound<'py, PyAny>, + key: &Bound<'py, PyAny>, + default: &str, +) -> PyResult> { + let value = optional_get(mapping, key)?; + if value.is_none() { + return Ok(string(mapping.py(), default)); + } + Ok(value) +} + +fn by_name<'py>(items: &Bound<'py, PyAny>) -> PyResult> { + let lookup = PyDict::new(items.py()); + for item in items.try_iter()? { + let item = item?; + lookup.set_item(item.getattr("name")?, item)?; + } + Ok(lookup) +} + +/// The plan reason for one selected seed. +fn seed_plan_reason<'py>( + seed_name: &Bound<'py, PyAny>, + expected_version_hash: &Bound<'py, PyAny>, + snapshot: &Bound<'py, PyAny>, + seed_plan_reasons: &Bound<'py, PyAny>, +) -> PyResult> { + let py = seed_name.py(); + let reason = optional_get(seed_plan_reasons, seed_name)?; + if !reason.is_none() { + return Ok(reason); + } + if !snapshot + .getattr("existing_relations")? + .contains(seed_name)? + { + return planner_type(py, "PlanReason", "FIRST_RUN"); + } + let fingerprint = snapshot + .getattr("fingerprints")? + .getattr("seeds")? + .call_method1("get", (seed_name,))?; + if fingerprint.is_none() { + return planner_type(py, "PlanReason", "FIRST_RUN"); + } + if !expected_version_hash.is_none() + && fingerprint + .getattr("version_hash")? + .ne(expected_version_hash)? + { + return planner_type(py, "PlanReason", "CONFIG_CHANGED"); + } + planner_type(py, "PlanReason", "NO_CHANGE") +} + +fn seed_entries<'py>( + project: &Bound<'py, PyAny>, + snapshot: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, + extras: &Bound<'py, PyAny>, +) -> PyResult>> { + let py = project.py(); + let version_hashes = extras.getattr("seed_version_hashes")?; + let metadata_jsons = extras.getattr("seed_metadata_jsons")?; + let plan_reasons = extras.getattr("seed_plan_reasons")?; + let mut entries: Vec> = Vec::new(); + for seed in project.getattr("seeds")?.try_iter()? { + let seed = seed?; + if !selected_keys.contains(seed.getattr("key")?)? { + continue; + } + let name = seed.getattr("name")?; + entries.push(planner_model( + py, + "SeedPlanEntry", + &[ + ("key", seed.getattr("key")?), + ("name", name.clone()), + ("destination", seed.getattr("destination")?), + ( + "file_path", + seed.getattr("seed_file")?.getattr("file_path")?, + ), + ( + "columns", + native(py, "extract_seed_columns", &[("seed", seed.clone())])?, + ), + ( + "csv_settings", + seed.getattr("schema_entry")?.getattr("csv_settings")?, + ), + ( + "fingerprint_definition", + optional_get_or(&metadata_jsons, &name, "")?, + ), + ( + "fingerprint_version_hash", + optional_get_or(&version_hashes, &name, "")?, + ), + ( + "fingerprint_metadata_json", + optional_get_or(&metadata_jsons, &name, "{}")?, + ), + ( + "reason", + seed_plan_reason( + &name, + &optional_get(&version_hashes, &name)?, + snapshot, + &plan_reasons, + )?, + ), + ], + )?); + } + Ok(entries) +} + +fn provider_usages<'py>( + provider_usages: &Bound<'py, PyAny>, + consumer_kind: &str, + consumer_name: &Bound<'py, PyAny>, +) -> PyResult>> { + let py = consumer_name.py(); + let mut usages: Vec> = Vec::new(); + for usage in provider_usages.try_iter()? { + let usage = usage?; + usages.push(planner_model( + py, + "PlanProviderUsage", + &[ + ("provider_name", usage.getattr("provider_name")?), + ("consumer_kind", string(py, consumer_kind)), + ("consumer_name", consumer_name.clone()), + ("parameter_name", usage.getattr("parameter_name")?), + ( + "annotation_class_name", + usage.getattr("annotation_class_name")?, + ), + ("annotation_module", usage.getattr("annotation_module")?), + ], + )?); + } + Ok(usages) +} + +fn hook_provider_usages<'py>( + entry: &Bound<'py, PyAny>, + hook_by_name: &Bound<'py, PyDict>, +) -> PyResult>> { + let py = entry.py(); + let mut usages: Vec> = Vec::new(); + let python_hook = imported(py, DISCOVERY_MODELS, "PythonHookEntry")?; + let mut hooks: Vec> = Vec::new(); + for field in ["pre_hooks", "post_hooks"] { + let hook_entry = entry.getattr(field)?; + if hook_entry.is_none() { + continue; + } + if hook_entry.is_instance_of::() || hook_entry.is_instance_of::() { + for hook in hook_entry.try_iter()? { + hooks.push(hook?); + } + } else { + hooks.push(hook_entry); + } + } + for hook in hooks { + if !hook.is_instance(&python_hook)? { + continue; + } + let Some(discovered) = hook_by_name.get_item(hook.getattr("name")?)? else { + continue; + }; + usages.extend(provider_usages( + &discovered.getattr("provider_usages")?, + HOOK_CONSUMER, + &discovered.getattr("name")?, + )?); + } + Ok(usages) +} + +/// Provider usages of the planned loaders, hooks and custom materializations, in plan order. +fn plan_provider_usages<'py>( + project: &Bound<'py, PyAny>, + model_entries: &Bound<'py, PyAny>, + source_load_entries: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let mut usages: Vec> = Vec::new(); + let loader_by_name = by_name(&project.getattr("loader_functions")?)?; + for entry in source_load_entries.try_iter()? { + let Some(loader) = loader_by_name.get_item(entry?.getattr("loader")?)? else { + continue; + }; + usages.extend(provider_usages( + &loader.getattr("provider_usages")?, + LOADER_CONSUMER, + &loader.getattr("name")?, + )?); + } + let hook_by_name = by_name(&project.getattr("hook_functions")?)?; + let materialization_by_name = by_name(&project.getattr("materialization_files")?)?; + for entry in model_entries.try_iter()? { + let entry = entry?; + usages.extend(hook_provider_usages(&entry, &hook_by_name)?); + let custom = entry.getattr("custom_materialization_name")?; + if custom.is_none() { + continue; + } + let Some(materialization) = materialization_by_name.get_item(custom)? else { + continue; + }; + usages.extend(provider_usages( + &materialization.getattr("provider_usages")?, + MATERIALIZATION_CONSUMER, + &materialization.getattr("name")?, + )?); + } + tuple_of(py, usages) +} + +/// Fail when a selected loader depends on an unselected intermediate loader with no target. +fn require_intermediate_loader_targets<'py>( + project: &Bound<'py, PyAny>, + snapshot: &Bound<'py, PyAny>, + source_map: &Bound<'py, PyAny>, + source_load_entries: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py = project.py(); + let loader_functions = project.getattr("loader_functions")?; + let loader_by_name = by_name(&loader_functions)?; + let selected_names = PyFrozenSet::new( + py, + source_load_entries + .try_iter()? + .map(|entry| entry?.getattr("name")) + .collect::>>()?, + )?; + let existing = snapshot.getattr("existing_relations")?; + for entry in source_load_entries.try_iter()? { + let source_entry = source_map.call_method1("get", (entry?.getattr("name")?,))?; + if source_entry.is_none() || source_entry.getattr("loader")?.is_none() { + continue; + } + let Some(loader_function) = loader_by_name.get_item(source_entry.getattr("loader")?)? + else { + continue; + }; + let dependencies = native( + py, + "upstream_loader_dependency_names", + &[ + ("loader_function", loader_function), + ("loader_functions", loader_functions.clone()), + ], + )?; + for dependency_name in dependencies.try_iter()? { + let dependency_name = dependency_name?; + let dependency = source_map.call_method1("get", (&dependency_name,))?; + if dependency.is_none() || selected_names.contains(dependency.getattr("name")?)? { + continue; + } + let table = dependency.getattr("table")?; + let target = if table.is_truthy()? { + table + } else { + dependency.getattr("name")? + }; + if existing.contains(&target)? { + continue; + } + let source_name = source_entry.getattr("name")?.str()?; + return Err(error_of( + py, + (PLANNER_EXCEPTIONS, "PlannerInputError"), + &format!( + "Source '{source_name}' requires intermediate loader '{}', but its target \ + relation '{}' does not exist; use +source:{source_name} to refresh upstream \ + ingress dependencies", + dependency_name.str()?, + target.str()?, + ), + &[], + )); + } + } + Ok(()) +} + +/// Build one canonical function entry from resolved relation inputs. +pub(super) fn function_entry<'py>( + function: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, + (fingerprint_query_sql, previous_query_sql, reason): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ), +) -> PyResult> { + let py = function.py(); + let body_sql = native( + py, + "resolve_function_sql", + &[ + ("adapter", adapter.clone()), + ("function", function.clone()), + ("model_locations", relations.getattr("model_locations")?), + ("seed_locations", relations.getattr("seed_locations")?), + ( + "function_locations", + relations.getattr("function_locations")?, + ), + ("source_map", relations.getattr("source_read_map")?), + ( + "source_warehouse_columns", + relations.getattr("source_warehouse_columns")?, + ), + ( + "star_exclude_keyword", + relations.getattr("star_exclude_keyword")?, + ), + ], + )?; + let field = |name: &str| function.getattr(name); + planner_model( + py, + "FunctionPlanEntry", + &[ + ("key", field("key")?), + ("name", field("name")?), + ("relative_path", field("relative_path")?), + ("destination", field("destination")?), + ("arguments", field("arguments")?), + ("returns", field("returns")?), + ("body_sql", body_sql), + ("fingerprint_query_sql", fingerprint_query_sql), + ("fingerprint_destination", field("fingerprint_destination")?), + ("return_columns", field("return_columns")?), + ("language", field("language")?), + ("source_file_path", field("source_file_path")?), + ("runtime_version", field("runtime_version")?), + ("entry_point", field("entry_point")?), + ("packages", field("packages")?), + ("previous_query_sql", previous_query_sql), + ("reason", reason), + ], + ) +} + +fn function_entries<'py>( + (project, adapter, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + relations: &Bound<'py, PyAny>, + changes: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult>> { + let py = project.py(); + let functions = changes.getattr("functions")?; + let previous = snapshot.getattr("fingerprints")?.getattr("functions")?; + let mut entries: Vec> = Vec::new(); + for function in project.getattr("functions")?.try_iter()? { + let function = function?; + if !selected_keys.contains(function.getattr("key")?)? { + continue; + } + let name = function.getattr("name")?; + let change = functions.get_item(&name)?; + let previous_query_sql = if previous.contains(&name)? { + previous.get_item(&name)?.getattr("definition")? + } else { + none(py) + }; + entries.push(function_entry( + &function, + adapter, + relations, + ( + change.getattr("fingerprint_sql")?, + previous_query_sql, + change.getattr("reason")?, + ), + )?); + } + Ok(entries) +} + +/// Whether a test is named by the selection or covers a selected resource, and not excluded. +pub(super) fn test_selected( + test: &Bound<'_, PyAny>, + selected_keys: &Bound<'_, PyAny>, + selection: &Bound<'_, PyAny>, +) -> PyResult { + let parent = test.getattr("parent_name")?; + let name = if parent.is_truthy()? { + parent + } else { + test.getattr("name")? + }; + if selection.getattr("excluded_names")?.contains(&name)? { + return Ok(false); + } + Ok(selection.getattr("names")?.contains(&name)? + || scope_overlaps(&test.getattr("scope_deps")?, selected_keys)?) +} + +/// Planned SQL test entries and the warnings planning them produced. +pub(super) struct SelectedTests<'py> { + pub(super) entries: Vec>, + pub(super) warnings: Vec>, +} + +/// Selected SQL test entries and their warnings, failing on invalid fixtures. +pub(super) fn selected_test_entries<'py>( + (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + selected_keys: &Bound<'py, PyAny>, + selection: &Bound<'py, PyAny>, + case_name: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let mut selected: Vec> = Vec::new(); + for test in project.getattr("sql_tests")?.try_iter()? { + let test = test?; + if !test_selected(&test, selected_keys, selection)? { + continue; + } + if !case_name.is_none() && test.getattr("case_name")?.ne(case_name)? { + continue; + } + selected.push(test); + } + let mut entries: Vec> = Vec::new(); + let mut warnings: Vec> = Vec::new(); + if selected.is_empty() { + return Ok(SelectedTests { entries, warnings }); + } + let analysis = project.getattr("settings")?.getattr("sql_analysis")?; + let fixture_context = if analysis.is_truthy()? { + native( + py, + "build_relation_fixture_context", + &[("project", project.clone())], + )? + } else { + none(py) + }; + let results = native( + py, + "plan_sql_tests", + &[ + ("tests", tuple_of(py, selected)?), + ("project", project.clone()), + ("adapter", adapter.clone()), + ("sql_analysis_enabled", analysis), + ( + "validate_fixtures", + pyo3::types::PyBool::new(py, true).to_owned().into_any(), + ), + ("fixture_planning_context", fixture_context), + ], + )?; + let mut diagnostics: Vec = Vec::new(); + for result in results.try_iter()? { + let result = result?; + let entry = result.getattr("entry")?; + if entry.is_none() { + for diagnostic in result.getattr("fixture_diagnostics")?.try_iter()? { + diagnostics.push(diagnostic?.str()?.to_string()); + } + continue; + } + entries.push(entry); + for warning in result.getattr("warnings")?.try_iter()? { + warnings.push(warning?); + } + } + if !diagnostics.is_empty() { + return Err(error_of( + py, + (PLANNER_EXCEPTIONS, "PlannerInputError"), + &format!("Invalid SQL test fixtures:\n- {}", diagnostics.join("\n- ")), + &[], + )); + } + Ok(SelectedTests { entries, warnings }) +} + +/// SQL tests selected by name or overlapping one selection, planned into a reusable result. +fn planned_sql_tests<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, + selection: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let selected = selected_test_entries((project, adapter), selected_keys, selection, &none(py))?; + planner_model( + py, + "PlannedSqlTests", + &[ + ("selected_keys", selected_keys.clone()), + ("sql_test_selection", selection.clone()), + ("entries", tuple_of(py, selected.entries)?), + ("warnings", tuple_of(py, selected.warnings)?), + ], + ) +} + +fn plan_output<'py>(args: &KeywordArguments<'py>) -> PyResult> { + let project = args.value("project")?; + let py = project.py(); + let adapter = args.value("adapter")?; + let scope = args.value("scope")?; + let snapshot = args.value("snapshot")?; + let relations = args.value("relations")?; + let model_entry_results = args.value("model_entry_results")?; + let extras = args.value("extras")?; + let extras = if extras.is_none() { + planner_model(py, "PlanOutputExtras", &[])? + } else { + extras + }; + let selected_keys = scope.getattr("selected_keys")?; + let execution_order = scope.getattr("execution_order")?; + let source_map = relations.getattr("source_map")?; + let seeds = seed_entries(&project, &snapshot, &selected_keys, &extras)?; + let source_load_entries = native( + py, + "build_source_load_entries", + &[ + ("execution_order", execution_order.clone()), + ("selected_keys", selected_keys.clone()), + ("source_map", source_map.clone()), + ("is_reload", args.value("reload_sources")?), + ], + )?; + require_intermediate_loader_targets(&project, &snapshot, &source_map, &source_load_entries)?; + let functions = function_entries( + (&project, &adapter, &snapshot), + &relations, + &args.value("changes")?, + &selected_keys, + )?; + let model_entries = model_entry_results.getattr("entries")?; + let materializations = PyDict::new(py); + for entry in model_entries.try_iter()? { + let entry = entry?; + materializations.set_item( + entry.getattr("name")?, + entry.getattr("materialization_type")?, + )?; + } + let audit_entries = native( + py, + "build_selected_audit_entries", + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("scope", scope.clone()), + ("relations", relations.clone()), + ("model_materializations", materializations.into_any()), + ], + )?; + let selection = scope.getattr("sql_test_selection")?; + let mut planned = extras.getattr("planned_sql_tests")?; + if planned.is_none() + || planned.getattr("selected_keys")?.ne(&selected_keys)? + || planned.getattr("sql_test_selection")?.ne(&selection)? + { + planned = planned_sql_tests(&project, &adapter, &selected_keys, &selection)?; + } + let test_entries = planned.getattr("entries")?; + let mut test_keys: Vec> = Vec::new(); + for entry in test_entries.try_iter()? { + test_keys.push(entry?.getattr("key")?); + } + let scoped_keys = selected_keys.call_method1("__or__", (PyFrozenSet::new(py, test_keys)?,))?; + let mut scoped_order: Vec> = Vec::new(); + for key in execution_order.try_iter()? { + let key = key?; + if scoped_keys.contains(&key)? { + scoped_order.push(key); + } + } + let mut warnings: Vec> = Vec::new(); + for warning in model_entry_results.getattr("warnings")?.try_iter()? { + warnings.push(warning?); + } + for warning in planned.getattr("warnings")?.try_iter()? { + warnings.push(warning?); + } + planner_model( + py, + "PlanOutput", + &[ + ("execution_order", tuple_of(py, scoped_order)?), + ("model_entries", model_entries.clone()), + ("seed_entries", tuple_of(py, seeds)?), + ( + "source_load_entries", + PyTuple::new( + py, + source_load_entries + .try_iter()? + .collect::>>()?, + )? + .into_any(), + ), + ("function_entries", tuple_of(py, functions)?), + ( + "audit_entries", + PyTuple::new(py, audit_entries.try_iter()?.collect::>>()?)? + .into_any(), + ), + ( + "test_entries", + PyTuple::new(py, test_entries.try_iter()?.collect::>>()?)? + .into_any(), + ), + ("selected_keys", selected_keys), + ("warnings", tuple_of(py, warnings)?), + ("upstream_deps", scope.getattr("upstream_deps")?), + ("downstream_deps", scope.getattr("downstream_deps")?), + ("model_locations", relations.getattr("model_locations")?), + ("seed_locations", relations.getattr("seed_locations")?), + ( + "function_locations", + relations.getattr("function_locations")?, + ), + ("source_map", source_map), + ("source_read_map", relations.getattr("source_read_map")?), + ( + "python_source_read_map", + relations.getattr("python_source_read_map")?, + ), + ("hook_functions", project.getattr("hook_functions")?), + ( + "enforce_explicit_references", + project.getattr("enforce_explicit_references")?, + ), + ( + "provider_usages", + plan_provider_usages(&project, &model_entries, &source_load_entries)?, + ), + ( + "python_identity_fingerprints", + snapshot.getattr("fingerprints")?.getattr("python_nodes")?, + ), + ], + ) +} + +/// Assemble the plan output from planned entries and resolved relation inputs. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn build_plan_output<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_plan_output", + kwargs, + ( + &[ + "project", + "adapter", + "scope", + "snapshot", + "relations", + "changes", + "model_entry_results", + "reload_sources", + ], + &["extras"], + ), + )?; + plan_output(&args) +} + +/// Build one canonical function entry from resolved relation inputs. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn plan_function<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "plan_function", + kwargs, + ( + &["function", "adapter", "relations", "fingerprint_query_sql"], + &["previous_query_sql", "reason"], + ), + )?; + let reason = if args.passed("reason")? { + args.value("reason")? + } else { + planner_type(py, "PlanReason", "NO_CHANGE")? + }; + function_entry( + &args.value("function")?, + &args.value("adapter")?, + &args.value("relations")?, + ( + args.value("fingerprint_query_sql")?, + args.value("previous_query_sql")?, + reason, + ), + ) +} + +/// Plan SQL tests selected by name or overlapping one selection into a reusable result. +#[pyfunction] +#[pyo3(signature = (*, project, adapter, selected_keys, sql_test_selection))] +fn plan_selected_sql_tests<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, + sql_test_selection: &Bound<'py, PyAny>, +) -> PyResult> { + planned_sql_tests(project, adapter, selected_keys, sql_test_selection) +} + +/// Plan selected SQL tests into `future`, which re-raises any failure on the joining thread. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn plan_selected_sql_tests_into<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult<()> { + let args = KeywordArguments::read( + py, + "plan_selected_sql_tests_into", + kwargs, + ( + &[ + "future", + "project", + "adapter", + "selected_keys", + "sql_test_selection", + ], + &[], + ), + )?; + let future = args.value("future")?; + match native( + py, + "plan_selected_sql_tests", + &[ + ("project", args.value("project")?), + ("adapter", args.value("adapter")?), + ("selected_keys", args.value("selected_keys")?), + ("sql_test_selection", args.value("sql_test_selection")?), + ], + ) { + Ok(planned) => future.call_method1("set_result", (planned,))?, + Err(error) => future.call_method1("set_exception", (error.value(future.py()),))?, + }; + Ok(()) +} + +/// Return model closures resolved natively in one project batch, keyed by SQL test key. +#[pyfunction] +#[pyo3(signature = (*, project, tests))] +fn sql_test_model_chain_names_by_key<'py>( + project: &Bound<'py, PyAny>, + tests: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let chains = native( + py, + "resolve_sql_test_model_chains", + &[("project", project.clone()), ("tests", tests.clone())], + )?; + let by_key = PyDict::new(py); + for (test, chain) in tests.try_iter()?.zip(chains.try_iter()?) { + by_key.set_item(test?.getattr("key")?, chain?)?; + } + Ok(by_key) +} + +/// Whether a test is named by the selection or covers a selected resource, and not excluded. +#[pyfunction] +#[pyo3(signature = (*, test, selected_keys, sql_test_selection))] +fn sql_test_is_selected( + test: &Bound<'_, PyAny>, + selected_keys: &Bound<'_, PyAny>, + sql_test_selection: &Bound<'_, PyAny>, +) -> PyResult { + test_selected(test, selected_keys, sql_test_selection) +} + +/// Selected SQL test entries and warnings, optionally narrowed to one parameterized case. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn build_selected_test_entries<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult<(Bound<'py, PyList>, Bound<'py, PyList>)> { + let args = KeywordArguments::read( + py, + "build_selected_test_entries", + kwargs, + ( + &["project", "adapter", "selected_keys", "sql_test_selection"], + &["case_name"], + ), + )?; + let selected = selected_test_entries( + (&args.value("project")?, &args.value("adapter")?), + &args.value("selected_keys")?, + &args.value("sql_test_selection")?, + &args.value("case_name")?, + )?; + Ok(( + PyList::new(py, selected.entries)?, + PyList::new(py, selected.warnings)?, + )) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_plan_output, module)?)?; + module.add_function(wrap_pyfunction!(plan_function, module)?)?; + module.add_function(wrap_pyfunction!(plan_selected_sql_tests, module)?)?; + module.add_function(wrap_pyfunction!(plan_selected_sql_tests_into, module)?)?; + module.add_function(wrap_pyfunction!(sql_test_is_selected, module)?)?; + module.add_function(wrap_pyfunction!(sql_test_model_chain_names_by_key, module)?)?; + module.add_function(wrap_pyfunction!(build_selected_test_entries, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs new file mode 100644 index 0000000000..32e77e42bf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs @@ -0,0 +1,13 @@ +//! Register plan entry and plan output assembly entry points. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + super::clone_plans::register(module)?; + super::command_plans::register(module)?; + super::display_plan::register(module)?; + super::execution_plan::register(module)?; + super::model_entries::register(module)?; + super::plan_output::register(module)?; + super::relations::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs new file mode 100644 index 0000000000..fd974efa8a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs @@ -0,0 +1,245 @@ +//! Relation locations and source metadata for plan entry construction. + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_entries::values::{native, none, planner_model}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +/// Relation keys needed to render selected audits without selecting those relations. +fn selected_audit_relation_keys<'py>( + project: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let keys = PySet::empty(py)?; + for audit in project.getattr("audits")?.try_iter()? { + let audit = audit?; + if native( + py, + "audit_is_selected", + &[ + ("audit", audit.clone()), + ("selected_keys", selected_keys.clone()), + ], + )? + .is_truthy()? + { + for dependency in audit.getattr("scope_deps")?.try_iter()? { + keys.add(dependency?)?; + } + } + } + Ok(PyFrozenSet::new(py, keys.iter())?.into_any()) +} + +/// Reuse already-gathered source columns for the read map, or gather them once. +fn source_warehouse_columns<'py>( + (project, adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + source_read_map: &Bound<'py, PyAny>, + known_source_columns: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py = project.py(); + if !known_source_columns.is_none() { + let known = PyDict::new(py); + for name in source_read_map.try_iter()? { + let name = name?; + if known_source_columns.contains(&name)? { + known.set_item(&name, known_source_columns.get_item(&name)?)?; + } + } + let names = PyFrozenSet::new(py, known.keys())?.into_any(); + return Ok((known.into_any(), names)); + } + let entries = PyTuple::new( + py, + source_read_map + .call_method0("values")? + .try_iter()? + .collect::>>()?, + )?; + let gathered = native( + py, + "gather_source_inspection", + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("connection", connection.clone()), + ("source_entries", entries.into_any()), + ], + )?; + Ok((gathered.get_item(0)?, gathered.get_item(1)?)) +} + +/// Resolve relation locations and source metadata for plan entry construction. +pub(crate) fn planner_relations_context<'py>( + args: &KeywordArguments<'py>, +) -> PyResult> { + let project = args.value("project")?; + let py = project.py(); + let adapter = args.value("adapter")?; + let connection = args.value("connection")?; + let scope = args.value("scope")?; + let deferral = args.value("deferral")?; + let deferral = if deferral.is_none() { + planner_model(py, "DeferralInputs", &[])? + } else { + deferral + }; + let project_config = args.value("project_config")?; + let local_config = args.value("local_config")?; + let selected_keys = scope.getattr("selected_keys")?; + let mut model_locations = native( + py, + "build_model_locations", + &[("models", project.getattr("models")?)], + )?; + let mut seed_locations = native( + py, + "build_seed_locations", + &[("seeds", project.getattr("seeds")?)], + )?; + let function_locations = native( + py, + "build_function_locations", + &[("functions", project.getattr("functions")?)], + )?; + let deferred_locations = deferral.getattr("deferred_locations")?; + if !deferred_locations.is_none() { + let applied = native( + py, + "apply_deferred_locations", + &[ + ("model_locations", model_locations), + ("seed_locations", seed_locations), + ("deferred_locations", deferred_locations), + ("selected_keys", selected_keys.clone()), + ], + )?; + model_locations = applied.get_item(0)?; + seed_locations = applied.get_item(1)?; + } + let audit_keys = selected_audit_relation_keys(&project, &selected_keys)?; + let relation_keys = args.value("relation_keys")?; + let load_keys = if relation_keys.is_none() { + selected_keys.call_method1("__or__", (&audit_keys,))? + } else { + relation_keys + }; + let source_map = native( + py, + "build_source_load_map", + &[ + ("project", project.clone()), + ("selected_keys", load_keys), + ( + "upstream_deps", + if scope.getattr("all_keys")?.is_truthy()? { + scope.getattr("upstream_deps")? + } else { + none(py) + }, + ), + ], + )?; + let deferral_enabled = deferral.getattr("source_deferral_enabled")?.is_truthy()?; + let read_map = |map: &Bound<'py, PyAny>, names: Option>| { + if !deferral_enabled { + return Ok(map.clone()); + } + let mut pairs = vec![ + ("project", project.clone()), + ("source_map", map.clone()), + ("selected_keys", selected_keys.clone()), + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ("defer_sources_to", deferral.getattr("defer_sources_to")?), + ]; + if let Some(names) = names { + pairs.push(("read_source_names", names)); + } + native(py, "build_source_read_map", &pairs) + }; + let source_read_map = read_map(&source_map, None)?; + let python_source_map = native( + py, + "with_declared_source_reads", + &[ + ("project", project.clone()), + ("source_map", source_map.clone()), + ("source_names", scope.getattr("python_read_source_names")?), + ], + )?; + let python_names = PyFrozenSet::new( + py, + python_source_map + .try_iter()? + .collect::>>()?, + )? + .into_any(); + let python_source_read_map = read_map(&python_source_map, Some(python_names))?; + let (columns, listed_source_names) = source_warehouse_columns( + (&project, &adapter, &connection), + &source_read_map, + &args.value("known_source_columns")?, + )?; + let semantic_columns = native( + py, + "get_semantic_source_columns", + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("connection", connection), + ("source_read_map", source_read_map.clone()), + ("selected_keys", selected_keys), + ("columns", columns), + ], + )?; + planner_model( + py, + "PlannerRelationsContext", + &[ + ("model_locations", model_locations), + ("seed_locations", seed_locations), + ("function_locations", function_locations), + ("source_map", source_map), + ("source_read_map", source_read_map), + ("python_source_read_map", python_source_read_map), + ("source_warehouse_columns", semantic_columns), + ( + "star_exclude_keyword", + adapter.call_method0("star_exclude_keyword")?, + ), + ("listed_source_names", listed_source_names), + ], + ) +} + +/// Resolve relation locations and source metadata for plan entry construction. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn build_planner_relations_context<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_planner_relations_context", + kwargs, + ( + &["project", "adapter", "connection", "scope"], + &[ + "deferral", + "project_config", + "local_config", + "known_source_columns", + "relation_keys", + ], + ), + )?; + planner_relations_context(&args) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_planner_relations_context, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs new file mode 100644 index 0000000000..694d1742f3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs @@ -0,0 +1,82 @@ +//! Python boundary values shared by plan entry and plan output assembly. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::PyString; + +pub(super) const NATIVE_MODULE: &str = "sqlbuild._native"; +pub(super) const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +pub(super) const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; +pub(super) const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; +pub(super) const PLANNER_CONSTANTS: &str = "sqlbuild.compiler.planner.constants"; +pub(super) const COMPILE_EXCEPTIONS: &str = "sqlbuild.compiler.compile.exceptions"; +pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; +pub(super) const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +pub(super) const DISCOVERY_MODELS: &str = "sqlbuild.compiler.discovery.models"; + +/// `sqlbuild._native.name(**pairs)`. +pub(super) fn native<'py>( + py: Python<'py>, + name: &str, + pairs: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with(&imported(py, NATIVE_MODULE, name)?, pairs) +} + +/// `sqlbuild.compiler.planner.models.name(**pairs)`. +pub(super) fn planner_model<'py>( + py: Python<'py>, + name: &str, + pairs: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with(&imported(py, PLANNER_MODELS, name)?, pairs) +} + +/// One member of a planner enumeration. +pub(super) fn planner_type<'py>( + py: Python<'py>, + class: &str, + member: &str, +) -> PyResult> { + imported(py, PLANNER_TYPES, class)?.getattr(member) +} + +/// A Python string. +pub(super) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { + PyString::new(py, text).into_any() +} + +/// A Python bool. +pub(super) fn boolean(py: Python<'_>, value: bool) -> PyResult> { + Ok(value.into_pyobject(py)?.to_owned().into_any()) +} + +/// `None`. +pub(super) fn none(py: Python<'_>) -> Bound<'_, PyAny> { + py.None().into_bound(py) +} + +/// `dataclasses.replace(object, **fields)`. +pub(super) fn replaced<'py>( + object: &Bound<'py, PyAny>, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + imported(object.py(), "dataclasses", "replace")? + .call((object,), Some(&keywords(object.py(), fields)?)) +} + +/// `class(message, **fields)` from `module`, as a raised error. +pub(super) fn error_of( + py: Python<'_>, + (module, class): (&str, &str), + message: &str, + fields: &[(&str, Bound<'_, PyAny>)], +) -> PyErr { + match imported(py, module, class) + .and_then(|class| class.call((message,), Some(&keywords(py, fields)?))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs index 7257d905c0..eb2556c564 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs @@ -292,7 +292,7 @@ fn topologically_order_keys<'py>( /// Model names reachable downstream from the given keys, including model roots. #[pyfunction] #[pyo3(signature = (*, start_keys, downstream_deps))] -fn build_downstream_model_name_closure<'py>( +pub(crate) fn build_downstream_model_name_closure<'py>( start_keys: &Bound<'py, PyAny>, downstream_deps: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs index 382d078218..ee7f791b5d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs @@ -31,6 +31,19 @@ pub(crate) fn sorted_dumps( dumps(&encoded, &dialect).map_err(|error| PyValueError::new_err(format!("{error:?}"))) } +/// `json.dumps(value, ensure_ascii=False, sort_keys=True, separators=separators)`. +pub(crate) fn unicode_sorted_dumps( + value: &Bound<'_, PyAny>, + separators: (&str, &str), +) -> PyResult { + let encoded: JsonValue = Encoder::new(value.py(), false)?.encode(value)?; + let mut options: StdlibJsonOptions = + StdlibJsonOptions::new(None).with_separators(separators.0, separators.1); + options.ensure_ascii = false; + dumps(&encoded, &JsonDialect::Stdlib(options)) + .map_err(|error| PyValueError::new_err(format!("{error:?}"))) +} + /// The C encoder's type dispatch, with dictionaries ordered as `sorted(dict.items())`. struct Encoder<'py> { default_str: bool, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index 200d78f652..761bc780a7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::planner_migrations::column_objects::{model, replace}; +use crate::bindings::_helpers::planner_migrations::known_input_columns::KnownInputColumns; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PySet, PySetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -66,17 +67,14 @@ fn plan_column_migrations<'py>( &state.getattr("inspect_column_events")?, &[("schemas", schemas.into_any())], )?; - let inputs = call_with( - &imported( - py, - "sqlbuild.compiler.planner.classes.known_input_columns", - "KnownInputColumns", - )?, - &[ - ("model_columns", snapshot.getattr("existing_columns")?), - ("source_columns", args.value("source_columns")?), - ], - )?; + let inputs = Bound::new( + py, + KnownInputColumns::new( + &snapshot.getattr("existing_columns")?, + &args.value("source_columns")?, + ), + )? + .into_any(); let mut entries: Vec> = Vec::new(); let mut warnings: Vec> = Vec::new(); let mut hints: Vec> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs new file mode 100644 index 0000000000..358b2d003c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs @@ -0,0 +1,50 @@ +//! Physical columns of the relations a model reads, as inspected by the planner. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyFrozenSet; +use pyo3::{pyclass, pymethods}; + +const SOURCE_INPUT_FUNCTION: &str = "__source"; +const REF_INPUT_FUNCTION: &str = "__ref"; + +/// Look up the lower-cased physical columns a query binds against, or None when unknown. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct KnownInputColumns { + model_columns: Py, + source_columns: Py, +} + +impl KnownInputColumns { + /// Inspected model and source columns by relation name. + pub(crate) fn new(model_columns: &Bound<'_, PyAny>, source_columns: &Bound<'_, PyAny>) -> Self { + Self { + model_columns: model_columns.clone().unbind(), + source_columns: source_columns.clone().unbind(), + } + } +} + +#[pymethods] +impl KnownInputColumns { + fn __call__<'py>( + &self, + py: Python<'py>, + kind: &str, + name: &str, + ) -> PyResult>> { + let inspected = match kind { + REF_INPUT_FUNCTION => self.model_columns.bind(py), + SOURCE_INPUT_FUNCTION => self.source_columns.bind(py), + _ => return Ok(None), + }; + let columns = inspected.call_method1("get", (name,))?; + if columns.is_none() || !columns.is_truthy()? { + return Ok(None); + } + let mut names: Vec = Vec::new(); + for column in columns.try_iter()? { + names.push(column?.getattr("name")?.extract::()?.to_lowercase()); + } + PyFrozenSet::new(py, names).map(Some) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index 1d488c57e2..a6a080315f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -13,6 +13,7 @@ mod discovery_models; mod discovery_requests; mod errors; mod fingerprint; +mod known_input_columns; mod matching; mod model_entries; mod model_handover; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs index 08dd8db18a..f80cc80940 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs @@ -7,8 +7,6 @@ pub(crate) const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exception pub(crate) const ADAPTER_MODELS: &str = "sqlbuild.adapter.contract.models"; pub(crate) const ADAPTER_TYPES: &str = "sqlbuild.adapter.contract.types"; pub(crate) const SPEC_MODELS: &str = "sqlbuild.spec.contracts.models"; -pub(crate) const PLAN_ENTRY_MODULE: &str = "sqlbuild.compiler.planner._helpers.output.plan_entry"; -pub(crate) const PLAN_OUTPUT_MODULE: &str = "sqlbuild.compiler.planner._helpers.output.plan_output"; pub(crate) const SNOWFLAKE_ADAPTER: &str = "snowflake"; pub(crate) const BIGQUERY_ADAPTER: &str = "bigquery"; pub(crate) const TRANSIENT_TABLE_TYPE: &str = "transient"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs index 4106139b51..e2f7805a21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs @@ -2,7 +2,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::planner_phases::constants::PLAN_ENTRY_MODULE; +use crate::bindings::_helpers::planner_phases::constants::NATIVE_MODULE; use crate::bindings::_helpers::planner_phases::objects::{native, planner_model}; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -127,7 +127,7 @@ pub(crate) fn build_planner_entry_results<'py>( let config = runtime.getattr("project_config")?; let execution_scope = pruning.getattr("execution_scope")?; let relations = call_with( - &imported(py, PLAN_ENTRY_MODULE, "build_planner_relations_context")?, + &imported(py, NATIVE_MODULE, "build_planner_relations_context")?, &[ ("project", project.clone()), ("adapter", runtime.getattr("adapter")?), @@ -202,7 +202,7 @@ pub(crate) fn build_planner_entry_results<'py>( ], )?; let entries = call_with( - &imported(py, PLAN_ENTRY_MODULE, "build_plan_entries")?, + &imported(py, NATIVE_MODULE, "build_plan_entries")?, &[ ("project", project), ("adapter", runtime.getattr("adapter")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index 99fb843502..ab65288a43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -2,7 +2,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::planner_phases::constants::PLAN_OUTPUT_MODULE; +use crate::bindings::_helpers::planner_phases::constants::NATIVE_MODULE; use crate::bindings::_helpers::planner_phases::objects::{ model_key_names, native, planner_model, replace, sorted_names, strings, text, tuple, }; @@ -92,7 +92,7 @@ pub(crate) fn assemble_base_plan_output<'py>( ], )?; let plan_output = call_with( - &imported(py, PLAN_OUTPUT_MODULE, "build_plan_output")?, + &imported(py, NATIVE_MODULE, "build_plan_output")?, &[ ("project", runtime.getattr("project")?), ("adapter", runtime.getattr("adapter")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs index d8cf02e822..c21f4b728a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -1,7 +1,7 @@ //! The warehouse inspection phase: snapshot, migrations and inspection relations in one pass. use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::planner_phases::constants::PLAN_ENTRY_MODULE; +use crate::bindings::_helpers::planner_phases::constants::NATIVE_MODULE; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; use crate::bindings::_helpers::planner_phases::objects::{native, planner_model}; use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; @@ -98,7 +98,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( )?; fingerprints.call_method0("persist")?; let inspection_relations = call_with( - &imported(py, PLAN_ENTRY_MODULE, "build_planner_relations_context")?, + &imported(py, NATIVE_MODULE, "build_planner_relations_context")?, &[ ("project", project.clone()), ("adapter", adapter.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs index cfffe8f312..ab366575e1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs @@ -3,8 +3,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::{ - planner_changes, planner_identity, planner_migrations, planner_phases, planner_resolution, - planner_scenario, planner_scope, planner_selection, planner_strategy, planner_warehouse, + planner_changes, planner_entries, planner_identity, planner_migrations, planner_phases, + planner_resolution, planner_scenario, planner_scope, planner_selection, planner_strategy, + planner_warehouse, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -18,5 +19,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { planner_scenario::registration::register(module)?; planner_strategy::registration::register(module)?; planner_resolution::registration::register(module)?; + planner_entries::registration::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs index 212257f4dd..cce4a7c1d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -4,6 +4,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::planner_resolution::constants::CURSOR_START; +use crate::bindings::_helpers::planner_resolution::cursor_policies::{ + future_cursor_config, start_cursor_config, +}; use crate::bindings::_helpers::planner_resolution::values::{NATIVE_MODULE, string}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; @@ -11,8 +14,6 @@ use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; const MODELS: &str = "sqlbuild.compiler.planner.models"; const CURSOR: &str = "sqlbuild._native"; -const POLICIES: &str = "sqlbuild.compiler.planner._helpers.resolve.cursor_policies"; -const FUTURE: &str = "sqlbuild.compiler.planner.main.execution.future_cursor_safety"; fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { call_with( @@ -103,20 +104,8 @@ fn compute_model_cursor_bounds<'py>( } let context = args.value("context")?; let incremental_mode = config(&model, "incremental_mode")?; - let future_config = call_with( - &imported(py, POLICIES, "resolve_future_cursor_config")?, - &[ - ("model", model.clone()), - ("project_config", context.getattr("future_cursor_config")?), - ], - )?; - let start_config = call_with( - &imported(py, POLICIES, "resolve_start_cursor_config")?, - &[ - ("model", model.clone()), - ("project_config", context.getattr("start_cursor_config")?), - ], - )?; + let future_config = future_cursor_config(&model, &context.getattr("future_cursor_config")?)?; + let start_config = start_cursor_config(&model, &context.getattr("start_cursor_config")?)?; if incremental_mode.eq("microbatch")? { return unbounded(py); } @@ -270,7 +259,7 @@ fn compute_model_cursor_bounds<'py>( return Ok(bounds); } call_with( - &imported(py, FUTURE, "apply_future_cursor_safety")?, + &imported(py, NATIVE_MODULE, "resolve_future_cursor_safety")?, &[ ("bounds", bounds), ("cursor_type", cursor_type), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs index a7d34b0743..5ef35e5179 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs @@ -21,7 +21,7 @@ fn type_cursor_overrides<'py>( py, "type_cursor_overrides", kwargs, - (&["cursor_overrides", "selected_models"], &[]), + (&["cursor_overrides"], &["selected_models"]), )?; let overrides = args.value("cursor_overrides")?; if overrides.is_none() || !overrides.getattr("has_untyped")?.is_truthy()? { @@ -29,7 +29,12 @@ fn type_cursor_overrides<'py>( } let get_config = imported(py, "sqlbuild._native", "get_config_str")?; let mut models: Vec<(String, String)> = Vec::new(); - for model in args.value("selected_models")?.try_iter()? { + let selected = if args.passed("selected_models")? { + args.value("selected_models")? + } else { + pyo3::types::PyTuple::empty(py).into_any() + }; + for model in selected.try_iter()? { let model = model?; let values = model.getattr("config")?.getattr("values")?; let config = |key: &str| { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs new file mode 100644 index 0000000000..f0bd2a13dc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs @@ -0,0 +1,106 @@ +//! Model-level cursor safety policy overrides over project configuration. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::PyString; +use pyo3::{pyfunction, wrap_pyfunction}; + +const SPEC_MODELS: &str = "sqlbuild.spec.contracts.models"; +const SPEC_TYPES: &str = "sqlbuild.spec.contracts.types"; +const POLICY_DISABLED: &str = "disabled"; + +/// The policy class, its distance field, and the model keys overriding distance and action. +struct PolicyFields { + class: &'static str, + distance: &'static str, + distance_key: &'static str, + action_key: &'static str, +} + +const FUTURE_POLICY: PolicyFields = PolicyFields { + class: "FutureCursorsConfig", + distance: "max_distance", + distance_key: "cursor_future_max_distance", + action_key: "cursor_future_action", +}; + +const START_POLICY: PolicyFields = PolicyFields { + class: "StartCursorsConfig", + distance: "max_ahead", + distance_key: "cursor_start_max_ahead", + action_key: "cursor_start_max_action", +}; + +fn resolve_policy<'py>( + model: &Bound<'py, PyAny>, + project_config: &Bound<'py, PyAny>, + fields: &PolicyFields, +) -> PyResult> { + let py = model.py(); + let class = imported(py, SPEC_MODELS, fields.class)?; + let configured = if project_config.is_truthy()? { + project_config.clone() + } else { + class.call0()? + }; + let values = model.getattr("config")?.getattr("values")?; + let raw_distance = values.call_method1("get", (fields.distance_key,))?; + let raw_action = values.call_method1("get", (fields.action_key,))?; + let mut distance = configured.getattr(fields.distance)?; + if raw_distance.is_instance_of::() { + distance = if raw_distance.eq(POLICY_DISABLED)? { + py.None().into_bound(py) + } else { + raw_distance + }; + } + let mut action = configured.getattr("action")?; + if raw_action.is_instance_of::() { + action = imported(py, SPEC_TYPES, "FutureCursorAction")?.call1((raw_action,))?; + } + if distance.is_none() { + return Ok(py.None().into_bound(py)); + } + call_with(&class, &[(fields.distance, distance), ("action", action)]) +} + +/// The future-end policy for one model, model configuration taking precedence. +pub(crate) fn future_cursor_config<'py>( + model: &Bound<'py, PyAny>, + project_config: &Bound<'py, PyAny>, +) -> PyResult> { + resolve_policy(model, project_config, &FUTURE_POLICY) +} + +/// The automatic-start policy for one model, model configuration taking precedence. +pub(crate) fn start_cursor_config<'py>( + model: &Bound<'py, PyAny>, + project_config: &Bound<'py, PyAny>, +) -> PyResult> { + resolve_policy(model, project_config, &START_POLICY) +} + +/// Resolve future-end policy with model config taking precedence. +#[pyfunction] +#[pyo3(signature = (*, model, project_config))] +fn resolve_future_cursor_config<'py>( + model: &Bound<'py, PyAny>, + project_config: &Bound<'py, PyAny>, +) -> PyResult> { + future_cursor_config(model, project_config) +} + +/// Resolve automatic-start policy with model config taking precedence. +#[pyfunction] +#[pyo3(signature = (*, model, project_config))] +fn resolve_start_cursor_config<'py>( + model: &Bound<'py, PyAny>, + project_config: &Bound<'py, PyAny>, +) -> PyResult> { + start_cursor_config(model, project_config) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_future_cursor_config, module)?)?; + module.add_function(wrap_pyfunction!(resolve_start_cursor_config, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs new file mode 100644 index 0000000000..dcf00e724a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs @@ -0,0 +1,712 @@ +//! Plan-entry cursor windows: overrides, planner-owned bounds and microbatch ranges. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::cursor::conversion::CursorSession; +use crate::bindings::_helpers::cursor::scalars::{parse_value, sentinel_to_token}; +use crate::bindings::_helpers::cursor::values::{local_date_time, replaced}; +use crate::bindings::_helpers::planner_resolution::cursor_bounds::duration; +use crate::bindings::_helpers::planner_resolution::cursor_policies::{ + future_cursor_config, start_cursor_config, +}; +use crate::bindings::_helpers::planner_resolution::cursor_safety::{replace, utc_clock}; +use crate::bindings::_helpers::planner_resolution::values::{NATIVE_MODULE, string}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; +use sqlbuild_cursor::algebra::main::compare::compare; +use sqlbuild_cursor::algebra::models::{Scalar, ScalarValue, Timestamp, ZoneOffset}; +use sqlbuild_cursor::planner::main::cursor_end_clamp::cursor_end_clamp; +use sqlbuild_cursor::planner::main::rolling_window::rolling_window; +use sqlbuild_cursor::planner::models::CursorEndClamp; +use std::cmp::Ordering; + +const MODELS: &str = "sqlbuild.compiler.planner.models"; +const CURSOR_TYPES: &str = "sqlbuild.cursor_algebra.types"; +const CURSOR_MODELS: &str = "sqlbuild.cursor_algebra.models"; +const COMPILE_EXCEPTIONS: &str = "sqlbuild.compiler.compile.exceptions"; +const INCREMENTAL: &str = "incremental"; +const MICROBATCH: &str = "microbatch"; +const ROLLING_WINDOW: &str = "rolling_window"; +const TIMESTAMP: &str = "timestamp"; +const INTEGER: &str = "integer"; +const BOUNDED_BACKFILL: &str = "bounded"; +const FULL_BACKFILL: &str = "full"; +const WINDOW_REQUIRED: [&str; 7] = [ + "model", + "snapshot", + "backfill", + "full_refresh", + "start_cursor_override", + "end_cursor_override", + "runtime_owned_cursor_bounds", +]; +const WINDOW_OPTIONAL: [&str; 5] = [ + "invocation_time", + "future_cursor_config", + "start_cursor_config", + "replaces_relation", + "cursor_input_relations", +]; + +/// The planner inputs that decide one model's plan-time cursor window. +pub(crate) struct CursorWindowInputs<'py> { + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) backfill: Bound<'py, PyAny>, + pub(crate) overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub(crate) runtime_owned: bool, + pub(crate) policies: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub(crate) invocation_time: Bound<'py, PyAny>, + pub(crate) full_refresh: bool, + pub(crate) replaces_relation: bool, + pub(crate) cursor_input_relations: Bound<'py, PyAny>, +} + +fn native<'py>( + py: Python<'py>, + name: &str, + pairs: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with(&imported(py, NATIVE_MODULE, name)?, pairs) +} + +/// One string-valued model configuration entry. +pub(crate) fn config_str<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { + native( + model.py(), + "get_config_str", + &[ + ("values", model.getattr("config")?.getattr("values")?), + ("key", string(model.py(), key)), + ], + ) +} + +/// One cursor-bound model configuration entry as its token text. +pub(crate) fn config_bound<'py>( + model: &Bound<'py, PyAny>, + key: &str, +) -> PyResult> { + native( + model.py(), + "get_config_cursor_bound", + &[ + ("values", model.getattr("config")?.getattr("values")?), + ("key", string(model.py(), key)), + ], + ) +} + +fn boolean(py: Python<'_>, value: bool) -> PyResult> { + Ok(value.into_pyobject(py)?.to_owned().into_any()) +} + +/// A cursor token as the scalar it orders by: integers for integer cursors, else instants. +fn order_scalar<'py>( + session: &CursorSession<'py>, + token: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult { + let py = session.py(); + let kind = if cursor_type.eq(INTEGER)? { + INTEGER + } else { + TIMESTAMP + }; + session.scalar(&parse_value(token, &string(py, kind))?) +} + +fn order<'py>( + session: &CursorSession<'py>, + tokens: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + cursor_type: &Bound<'py, PyAny>, +) -> PyResult { + let left = order_scalar(session, tokens.0, cursor_type)?; + let right = order_scalar(session, tokens.1, cursor_type)?; + session.checked(compare(&left, &right, session)) +} + +fn token<'py>(py: Python<'py>, bound: &Bound<'py, PyAny>) -> PyResult> { + Ok(PyString::new(py, &sentinel_to_token(bound)?).into_any()) +} + +fn sentinel<'py>(py: Python<'py>, name: &str) -> PyResult> { + imported(py, CURSOR_TYPES, "BoundSentinel")?.getattr(name) +} + +/// `bounds` clamped so a concrete window never reaches the exclusive configured `cursor_end`. +fn clamp_cursor_end<'py>( + bounds: Bound<'py, PyAny>, + model: &Bound<'py, PyAny>, +) -> PyResult> { + let py = model.py(); + let cursor_end = config_bound(model, "cursor_end")?; + let cursor_type = config_str(model, "cursor_type")?; + if bounds.is_none() || cursor_end.is_none() { + return Ok(bounds); + } + let start = bounds.getattr("start")?; + let end = bounds.getattr("end")?; + if start.eq(sentinel(py, "START")?)? || end.eq(sentinel(py, "END")?)? { + return Ok(bounds); + } + let session = CursorSession::new(py)?; + let window = ( + order_scalar(&session, &token(py, &start)?, &cursor_type)?, + order_scalar(&session, &token(py, &end)?, &cursor_type)?, + ); + let limit = order_scalar(&session, &cursor_end, &cursor_type)?; + match session.checked(cursor_end_clamp((&window.0, &window.1), &limit, &session))? { + CursorEndClamp::Unchanged => Ok(bounds), + CursorEndClamp::Empty => replace( + &bounds, + &[("start", cursor_end.clone()), ("end", cursor_end)], + ), + CursorEndClamp::Capped => replace(&bounds, &[("end", cursor_end)]), + } +} + +fn timestamp_value<'py>(clock: &Bound<'py, PyAny>, scalar: &Scalar) -> PyResult> { + let ScalarValue::Timestamp(timestamp) = &scalar.value else { + return Ok(clock.py().None().into_bound(clock.py())); + }; + call_with( + &imported(clock.py(), CURSOR_MODELS, "TimestampValue")?, + &[("value", replaced(clock, ×tamp.local, timestamp.fold)?)], + ) +} + +/// The invocation-anchored rolling microbatch window, or `None` without a grain or lookback. +fn rolling_window_bounds<'py>( + model: &Bound<'py, PyAny>, + invocation_time: &Bound<'py, PyAny>, +) -> PyResult> { + let py = model.py(); + let grain = config_str(model, "cursor_grain")?; + let lookback = native( + py, + "resolve_microbatch_lookback", + &[("model", model.clone()), ("effective_grain", grain.clone())], + )?; + let lookback = if lookback.is_truthy()? { + lookback + } else { + string(py, "") + }; + let span = duration(&lookback)?; + let (Some(span), false) = (span, grain.is_none()) else { + return Ok(py.None().into_bound(py)); + }; + let utc = imported(py, "datetime", "UTC")?; + let moment = if invocation_time.is_none() { + imported(py, "datetime", "datetime")?.call_method1("now", (&utc,))? + } else { + invocation_time.clone() + }; + let moment = if moment.getattr("tzinfo")?.is_none() { + call_with(&moment.getattr("replace")?, &[("tzinfo", utc)])? + } else { + utc_clock(&moment)? + }; + let session = CursorSession::new(py)?; + let Some(grain) = session.grain(&grain)? else { + return Ok(py.None().into_bound(py)); + }; + let clock = Scalar { + value: ScalarValue::Timestamp(Timestamp { + local: local_date_time(&moment)?, + fold: moment.getattr("fold")?.extract::()? != 0, + offset: ZoneOffset::Fixed(0), + template: None, + source_text: None, + }), + handle: None, + }; + let (start, end) = session.checked(rolling_window(&clock, &span, grain))?; + let mut bounds = call_with( + &imported(py, MODELS, "CursorBounds")?, + &[ + ("start", timestamp_value(&moment, &start)?), + ("end", timestamp_value(&moment, &end)?), + ], + )?; + let cursor_start = config_bound(model, "cursor_start")?; + let timestamp = string(py, TIMESTAMP); + if !cursor_start.is_none() + && order( + &session, + (&token(py, &bounds.getattr("start")?)?, &cursor_start), + ×tamp, + )? == Ordering::Less + { + bounds = replace(&bounds, &[("start", cursor_start)])?; + } + clamp_cursor_end(bounds, model) +} + +fn apply_future_safety<'py>( + bounds: Bound<'py, PyAny>, + typing: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + inputs: &CursorWindowInputs<'py>, + input_evidence: Bound<'py, PyAny>, +) -> PyResult> { + let py = bounds.py(); + if bounds.is_none() || bounds.getattr("end")?.eq(sentinel(py, "END")?)? { + return Ok(bounds); + } + native( + py, + "resolve_future_cursor_safety", + &[ + ("bounds", bounds), + ("cursor_type", typing.0.clone()), + ("cursor_grain", typing.1.clone()), + ("config", inputs.policies.0.clone()), + ("invocation_time", inputs.invocation_time.clone()), + ( + "has_complete_override", + boolean( + py, + !inputs.overrides.0.is_none() && !inputs.overrides.1.is_none(), + )?, + ), + ("input_evidence", input_evidence), + ], + ) +} + +fn bounded_override<'py>( + model: &Bound<'py, PyAny>, + inputs: &CursorWindowInputs<'py>, +) -> PyResult> { + native( + model.py(), + "resolve_bounded_cursor_override", + &[ + ("start_cursor_override", inputs.overrides.0.clone()), + ("end_cursor_override", inputs.overrides.1.clone()), + ("cursor_type", config_str(model, "cursor_type")?), + ("cursor_grain", config_str(model, "cursor_grain")?), + ], + ) +} + +fn backfill_duration<'py>(backfill: &Bound<'py, PyAny>) -> PyResult> { + if backfill.getattr("action")?.eq(BOUNDED_BACKFILL)? { + return backfill.getattr("duration"); + } + Ok(backfill.py().None().into_bound(backfill.py())) +} + +/// Cursor bounds and the maximum-start policy inputs, shared by both window computations. +struct BoundsRequest<'py> { + cursor_snapshot: Bound<'py, PyAny>, + typing: (Bound<'py, PyAny>, Bound<'py, PyAny>), + lookback: Bound<'py, PyAny>, + is_microbatch: bool, +} + +fn snapshot_bounds<'py>( + model: &Bound<'py, PyAny>, + inputs: &CursorWindowInputs<'py>, + request: BoundsRequest<'py>, +) -> PyResult> { + let py = model.py(); + let policy = call_with( + &imported(py, MODELS, "MaximumStartPolicyInputs")?, + &[ + ("config", inputs.policies.1.clone()), + ("invocation_time", inputs.invocation_time.clone()), + ( + "incremental_strategy", + config_str(model, "incremental_strategy")?, + ), + ("incremental_mode", config_str(model, "incremental_mode")?), + ], + )?; + let bounds = native( + py, + "compute_cursor_bounds", + &[ + ("cursor_snapshot", request.cursor_snapshot.clone()), + ("cursor_type", request.typing.0.clone()), + ("cursor_start", config_bound(model, "cursor_start")?), + ("lookback", request.lookback), + ("backfill_duration", backfill_duration(&inputs.backfill)?), + ("start_cursor_override", inputs.overrides.0.clone()), + ("end_cursor_override", inputs.overrides.1.clone()), + ("is_microbatch", boolean(py, request.is_microbatch)?), + ("cursor_grain", request.typing.1.clone()), + ("maximum_start_policy", policy), + ], + )?; + let bounds = apply_future_safety( + bounds, + (&request.typing.0, &request.typing.1), + inputs, + request.cursor_snapshot.getattr("input_evidence")?, + )?; + clamp_cursor_end(bounds, model) +} + +fn model_cursor_snapshot<'py>( + model: &Bound<'py, PyAny>, + snapshot: &Bound<'py, PyAny>, +) -> PyResult> { + snapshot + .getattr("cursor_snapshots")? + .call_method1("get", (model.getattr("name")?,)) +} + +fn without_destination<'py>(cursor_snapshot: Bound<'py, PyAny>) -> PyResult> { + native( + cursor_snapshot.py(), + "without_destination_cursor", + &[("cursor_snapshot", cursor_snapshot)], + ) +} + +/// Compute cursor bounds for inclusion on the plan entry. +pub(crate) fn plan_cursor_bounds<'py>( + model: &Bound<'py, PyAny>, + inputs: &CursorWindowInputs<'py>, +) -> PyResult> { + let py = model.py(); + if !config_str(model, "materialized")?.eq(INCREMENTAL)? + || config_str(model, "cursor")?.is_none() + { + return Ok(py.None().into_bound(py)); + } + let bounded = bounded_override(model, inputs)?; + if !bounded.is_none() { + return clamp_cursor_end(bounded, model); + } + if inputs.runtime_owned || inputs.full_refresh { + return Ok(py.None().into_bound(py)); + } + let mut cursor_snapshot = model_cursor_snapshot(model, &inputs.snapshot)?; + if cursor_snapshot.is_none() { + return Ok(cursor_snapshot); + } + if inputs.backfill.getattr("action")?.eq(FULL_BACKFILL)? { + cursor_snapshot = without_destination(cursor_snapshot)?; + } + snapshot_bounds( + model, + inputs, + BoundsRequest { + cursor_snapshot, + typing: ( + config_str(model, "cursor_type")?, + config_str(model, "cursor_grain")?, + ), + lookback: config_str(model, "lookback")?, + is_microbatch: config_str(model, "incremental_mode")?.eq(MICROBATCH)?, + }, + ) +} + +/// Compute the real overall cursor range for microbatch batch splitting. +pub(crate) fn microbatch_range<'py>( + model: &Bound<'py, PyAny>, + inputs: &CursorWindowInputs<'py>, +) -> PyResult> { + let py = model.py(); + if !config_str(model, "materialized")?.eq(INCREMENTAL)? + || !config_str(model, "incremental_mode")?.eq(MICROBATCH)? + || config_str(model, "cursor")?.is_none() + { + return Ok(py.None().into_bound(py)); + } + let bounded = bounded_override(model, inputs)?; + if !bounded.is_none() { + return clamp_cursor_end(bounded, model); + } + let strategy = config_str(model, "microbatch_strategy")?; + if strategy.eq(ROLLING_WINDOW)? { + let rolling = rolling_window_bounds(model, &inputs.invocation_time)?; + if inputs.full_refresh && !rolling.is_none() { + let cursor_start = config_bound(model, "cursor_start")?; + if !cursor_start.is_none() { + return replace(&rolling, &[("start", cursor_start)]); + } + } + return Ok(rolling); + } + if inputs.runtime_owned { + return Ok(py.None().into_bound(py)); + } + let mut cursor_snapshot = model_cursor_snapshot(model, &inputs.snapshot)?; + if cursor_snapshot.is_none() { + return Ok(cursor_snapshot); + } + if inputs.replaces_relation && !inputs.full_refresh { + cursor_snapshot = without_destination(cursor_snapshot)?; + } + let cursor_type = config_str(model, "cursor_type")?; + let mut grains: Vec> = Vec::new(); + for relation in inputs.cursor_input_relations.try_iter()? { + grains.push(relation?.getattr("cursor_grain")?); + } + let effective_grain = native( + py, + "resolve_effective_timestamp_grain", + &[ + ("cursor_type", cursor_type.clone()), + ("downstream_grain", config_str(model, "cursor_grain")?), + ("microbatch_strategy", strategy), + ("cursor_input_grains", PyTuple::new(py, grains)?.into_any()), + ], + )?; + if cursor_type.eq(TIMESTAMP)? && !effective_grain.is_none() { + cursor_snapshot = native( + py, + "normalize_cursor_snapshot_grain", + &[ + ("cursor_snapshot", cursor_snapshot), + ("cursor_type", cursor_type.clone()), + ("effective_grain", effective_grain.clone()), + ], + )?; + } + let lookback = native( + py, + "resolve_microbatch_lookback", + &[ + ("model", model.clone()), + ("effective_grain", effective_grain.clone()), + ], + )?; + snapshot_bounds( + model, + inputs, + BoundsRequest { + cursor_snapshot, + typing: (cursor_type, effective_grain), + lookback, + is_microbatch: false, + }, + ) +} + +/// Empty inputs that leave a planner-owned window empty, and whether others have rows. +pub(crate) fn empty_cursor_inputs<'py>( + model: &Bound<'py, PyAny>, + inputs: &CursorWindowInputs<'py>, +) -> PyResult<(Bound<'py, PyAny>, bool)> { + let py = model.py(); + let nothing = + || -> PyResult<(Bound<'py, PyAny>, bool)> { Ok((PyTuple::empty(py).into_any(), false)) }; + if inputs.full_refresh + || inputs.runtime_owned + || !config_str(model, "materialized")?.eq(INCREMENTAL)? + || config_str(model, "cursor")?.is_none() + || config_str(model, "microbatch_strategy")?.eq(ROLLING_WINDOW)? + || (!inputs.overrides.0.is_none() && !inputs.overrides.1.is_none()) + { + return nothing(); + } + let cursor_snapshot = model_cursor_snapshot(model, &inputs.snapshot)?; + if cursor_snapshot.is_none() { + return nothing(); + } + let decision = native( + py, + "cursor_inputs_without_rows", + &[("cursor_snapshot", cursor_snapshot)], + )?; + Ok(( + decision.getattr("input_names")?, + decision.getattr("waiting_on_empty_inputs")?.is_truthy()?, + )) +} + +/// The future-end and automatic-start policies effective for one model. +pub(crate) fn cursor_safety_configs<'py>( + model: &Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + Ok(( + future_cursor_config(model, &context.getattr("future_cursor_config")?)?, + start_cursor_config(model, &context.getattr("start_cursor_config")?)?, + )) +} + +fn compile_input_error(py: Python<'_>, message: String) -> PyErr { + match imported(py, COMPILE_EXCEPTIONS, "CompileInputError") + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// Resolve typed cursor overrides to generic start/end for one model. +pub(crate) fn model_cursor_overrides<'py>( + model: &Bound<'py, PyAny>, + cursor_overrides: &Bound<'py, PyAny>, + (start, end): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py = model.py(); + let cursor_type = config_str(model, "cursor_type")?; + let (mut resolved_start, mut resolved_end) = (start, end); + let typed_fields = if cursor_overrides.is_none() { + None + } else if cursor_type.eq(TIMESTAMP)? { + Some(("start_ts", "end_ts")) + } else if cursor_type.eq(INTEGER)? { + Some(("start_int", "end_int")) + } else { + None + }; + if let Some((start_field, end_field)) = typed_fields { + let typed_start = cursor_overrides.getattr(start_field)?; + if !typed_start.is_none() { + resolved_start = token(py, &typed_start)?; + } + let typed_end = cursor_overrides.getattr(end_field)?; + if !typed_end.is_none() { + resolved_end = token(py, &typed_end)?; + } + } + let session = CursorSession::new(py)?; + let name = model.getattr("name")?.str()?; + let cursor_start = config_bound(model, "cursor_start")?; + let cursor_end = config_bound(model, "cursor_end")?; + if !resolved_start.is_none() + && !cursor_start.is_none() + && order(&session, (&resolved_start, &cursor_start), &cursor_type)? == Ordering::Less + { + return Err(compile_input_error( + py, + format!( + "model '{name}': explicit start {} is before cursor_start {}", + resolved_start.str()?, + cursor_start.str()? + ), + )); + } + if !cursor_end.is_none() { + for (value, label) in [(&resolved_start, "start"), (&resolved_end, "end")] { + if !value.is_none() + && order(&session, (value, &cursor_end), &cursor_type)? != Ordering::Less + { + return Err(compile_input_error( + py, + format!( + "model '{name}': explicit {label} {} is at or beyond exclusive cursor_end {}", + value.str()?, + cursor_end.str()? + ), + )); + } + } + } + Ok((resolved_start, resolved_end)) +} + +fn window_inputs<'py>(args: &KeywordArguments<'py>) -> PyResult> { + Ok(CursorWindowInputs { + snapshot: args.value("snapshot")?, + backfill: args.value("backfill")?, + overrides: ( + args.value("start_cursor_override")?, + args.value("end_cursor_override")?, + ), + runtime_owned: args.flag("runtime_owned_cursor_bounds")?, + policies: ( + args.value("future_cursor_config")?, + args.value("start_cursor_config")?, + ), + invocation_time: args.value("invocation_time")?, + full_refresh: args.flag("full_refresh")?, + replaces_relation: args.flag("replaces_relation")?, + cursor_input_relations: if args.passed("cursor_input_relations")? { + args.value("cursor_input_relations")? + } else { + PyTuple::empty(args.value("snapshot")?.py()).into_any() + }, + }) +} + +/// Compute cursor bounds for inclusion on the plan entry. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn compute_plan_cursor_bounds<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "compute_plan_cursor_bounds", + kwargs, + (&WINDOW_REQUIRED, &WINDOW_OPTIONAL), + )?; + plan_cursor_bounds(&args.value("model")?, &window_inputs(&args)?) +} + +/// Compute the real overall cursor range for microbatch batch splitting. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn compute_microbatch_range<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "compute_microbatch_range", + kwargs, + (&WINDOW_REQUIRED, &WINDOW_OPTIONAL), + )?; + microbatch_range(&args.value("model")?, &window_inputs(&args)?) +} + +/// Return empty inputs that leave a planner-owned window empty, and whether others have rows. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn resolve_empty_cursor_inputs<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult<(Bound<'py, PyAny>, bool)> { + let args = KeywordArguments::read( + py, + "resolve_empty_cursor_inputs", + kwargs, + ( + &[ + "model", + "snapshot", + "full_refresh", + "runtime_owned_cursor_bounds", + "start_cursor_override", + "end_cursor_override", + ], + &[], + ), + )?; + empty_cursor_inputs(&args.value("model")?, &window_inputs(&args)?) +} + +/// Resolve typed cursor overrides to generic start/end for one model. +#[pyfunction] +#[pyo3(signature = (*, model, cursor_overrides, start_cursor_override, end_cursor_override))] +fn resolve_cursor_overrides<'py>( + model: &Bound<'py, PyAny>, + cursor_overrides: &Bound<'py, PyAny>, + start_cursor_override: Bound<'py, PyAny>, + end_cursor_override: Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + model_cursor_overrides( + model, + cursor_overrides, + (start_cursor_override, end_cursor_override), + ) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(compute_plan_cursor_bounds, module)?)?; + module.add_function(wrap_pyfunction!(compute_microbatch_range, module)?)?; + module.add_function(wrap_pyfunction!(resolve_empty_cursor_inputs, module)?)?; + module.add_function(wrap_pyfunction!(resolve_cursor_overrides, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs new file mode 100644 index 0000000000..07ba97a0c5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs @@ -0,0 +1,115 @@ +//! Prominent warnings for applied future-end and automatic-start cursor safety. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cursor::scalars::render_value; +use crate::bindings::_helpers::planner_resolution::values::string; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +const MODELS: &str = "sqlbuild.compiler.planner.models"; +const TYPES: &str = "sqlbuild.compiler.planner.types"; +const FUTURE_CAPPED_CODE: &str = "FUTURE_CURSOR_CAPPED"; +const START_CAPPED_CODE: &str = "AUTOMATIC_START_CAPPED"; + +fn rendered(evidence: &Bound<'_, PyAny>, field: &str) -> PyResult { + render_value(&evidence.getattr(field)?) +} + +/// Render a prominent warning for applied future cursor policy. +pub(crate) fn future_cap_message(bounds: &Bound<'_, PyAny>) -> PyResult> { + if bounds.is_none() { + return Ok(None); + } + let evidence = bounds.getattr("future_safety")?; + if evidence.is_none() { + return Ok(None); + } + Ok(Some(format!( + "FUTURE CURSOR CAPPED: discovered bounds ['{}', '{}'), applied bounds ['{}', '{}').", + rendered(&evidence, "discovered_start")?, + rendered(&evidence, "discovered_end")?, + rendered(&evidence, "applied_start")?, + rendered(&evidence, "applied_end")?, + ))) +} + +/// Render a warning that distinguishes start recovery from future-end capping. +pub(crate) fn maximum_start_message(bounds: &Bound<'_, PyAny>) -> PyResult> { + if bounds.is_none() { + return Ok(None); + } + let evidence = bounds.getattr("maximum_start_safety")?; + if evidence.is_none() { + return Ok(None); + } + let highest = evidence.getattr("highest_eligible_target_max")?; + let highest = if highest.is_none() { + "None".to_owned() + } else { + PyString::new(bounds.py(), &render_value(&highest)?) + .repr()? + .extract()? + }; + Ok(Some(format!( + "AUTOMATIC START CAPPED: physical target MAX '{}' exceeded horizon '{}'; highest \ + eligible target MAX={highest}, effective post-lookback start='{}', action={}, \ + input={}.{}.", + rendered(&evidence, "physical_target_max")?, + rendered(&evidence, "maximum_allowed_start")?, + rendered(&evidence, "effective_start")?, + evidence.getattr("action")?.getattr("value")?.str()?, + evidence.getattr("target_relation")?.str()?, + evidence.getattr("cursor_column")?.str()?, + ))) +} + +/// `warnings` followed by the future-end and automatic-start warnings `bounds` carries. +pub(crate) fn with_cursor_safety_warnings<'py>( + warnings: &Bound<'py, PyAny>, + model_name: &Bound<'py, PyAny>, + bounds: &Bound<'py, PyAny>, +) -> PyResult> { + let py = warnings.py(); + let mut all: Vec> = warnings + .try_iter()? + .collect::>>>()?; + for (message, code) in [ + (future_cap_message(bounds)?, FUTURE_CAPPED_CODE), + (maximum_start_message(bounds)?, START_CAPPED_CODE), + ] { + let Some(message) = message else { continue }; + all.push(call_with( + &imported(py, MODELS, "PlanWarning")?, + &[ + ("model_name", model_name.clone()), + ( + "severity", + imported(py, TYPES, "WarningSeverity")?.getattr("WARNING")?, + ), + ("code", string(py, code)), + ("message", string(py, &message)), + ], + )?); + } + Ok(PyTuple::new(py, all)?.into_any()) +} + +/// Render a prominent warning for applied future cursor policy. +#[pyfunction] +#[pyo3(signature = (bounds, /))] +fn future_cursor_cap_warning(bounds: &Bound<'_, PyAny>) -> PyResult> { + future_cap_message(bounds) +} + +/// Render a warning that distinguishes start recovery from future-end capping. +#[pyfunction] +#[pyo3(signature = (bounds, /))] +fn maximum_start_cap_warning(bounds: &Bound<'_, PyAny>) -> PyResult> { + maximum_start_message(bounds) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(future_cursor_cap_warning, module)?)?; + module.add_function(wrap_pyfunction!(maximum_start_cap_warning, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs index b90f4dc22a..6fa15a9dae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs @@ -4,7 +4,10 @@ mod constants; mod cursor_bounds; mod cursor_model; mod cursor_overrides; +pub(crate) mod cursor_policies; +pub(crate) mod cursor_ranges; mod cursor_safety; +pub(crate) mod cursor_warnings; mod cursor_windows; mod future_safety; mod locations; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index fe8a260dbd..f6ec9df817 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -3,13 +3,12 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::planner_resolution::values::{NATIVE_MODULE, string}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; -const CONFIG_MODULE: &str = "sqlbuild.compiler.planner._helpers.resolve.config"; -const INTRINSICS_MODULE: &str = "sqlbuild.compiler.planner._helpers.resolve.cursor_intrinsics"; const MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const HOOK_MODULE: &str = "sqlbuild.compiler.discovery.models"; const ERROR_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; @@ -213,8 +212,7 @@ fn resolve_model_sql<'py>( &imported(py, NATIVE_MODULE, "resolve_cursor_input_roles")?, &[("model", model.clone())], )?; - let inclusive = - imported(py, CONFIG_MODULE, "get_config_append_cursor_inclusive")?.call1((&model,))?; + let inclusive = append_cursor_inclusive(&model)?; let mut sql = resolution.resolve( model.getattr("query_sql")?, &[ @@ -241,20 +239,64 @@ fn resolve_model_sql<'py>( }; return Err(unresolved_bounds_error(py, &name, cause.as_deref())?); } - sql = call_with( - &imported(py, INTRINSICS_MODULE, "render_cursor_intrinsic_bounds")?, - &[ - ("sql", sql), - ("bounds", bounds), - ("cursor_type", cursor_type), - ("adapter", resolution.adapter.clone()), - ], - )?; + sql = intrinsic_bounds(&sql, &bounds, &cursor_type, &resolution.adapter)?; } resolution.assert_resolved(&sql, &format!("Model '{name}'"))?; Ok(sql) } +/// Effective append cursor lower-bound inclusivity. +fn append_cursor_inclusive<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py = model.py(); + let strategy = call_with( + &imported(py, NATIVE_MODULE, "get_config_str")?, + &[ + ("values", model.getattr("config")?.getattr("values")?), + ("key", string(py, "incremental_strategy")), + ], + )?; + let raw = model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("append_cursor_inclusive",))?; + let inclusive = + !strategy.eq("append")? || !raw.is_instance_of::() || raw.is_truthy()?; + Ok(inclusive.into_pyobject(py)?.to_owned().into_any()) +} + +/// `__cursor_start()` / `__cursor_end()` replaced with the adapter's typed bound literals. +fn intrinsic_bounds<'py>( + sql: &Bound<'py, PyAny>, + bounds: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py = sql.py(); + let literal = |field: &str| -> PyResult> { + call_with( + &adapter.getattr("render_cursor_bound_literal")?, + &[ + ( + "value", + PyString::new(py, &sentinel_to_token(&bounds.getattr(field)?)?).into_any(), + ), + ("cursor_type", cursor_type.clone()), + ], + ) + }; + let start_sql = literal("start")?; + let end_sql = literal("end")?; + call_with( + &imported(py, NATIVE_MODULE, "resolve_cursor_intrinsics")?, + &[ + ("sql", sql.clone()), + ("start_sql", start_sql), + ("end_sql", end_sql), + ], + )? + .get_item(0) +} + fn unresolved_bounds_error(py: Python<'_>, name: &str, cause: Option<&str>) -> PyResult { let error = imported(py, ERROR_MODULE, "PlannerInputError")?; let kwargs = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs index 6626407126..94a90504d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs @@ -6,6 +6,9 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::cursor_bounds::register(module)?; super::cursor_model::register(module)?; super::cursor_overrides::register(module)?; + super::cursor_policies::register(module)?; + super::cursor_ranges::register(module)?; + super::cursor_warnings::register(module)?; super::future_safety::register(module)?; super::maximum_safety::register(module)?; super::cursor_windows::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs index 1cf94c2c32..22748240c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs @@ -74,11 +74,7 @@ pub(super) fn function_entry<'py>( ) -> PyResult> { let py = function.py(); let body = call_with( - &imported( - py, - "sqlbuild.compiler.planner._helpers.resolve.resolve", - "resolve_function_sql", - )?, + &imported(py, "sqlbuild._native", "resolve_function_sql")?, &[ ("adapter", context.adapter.clone()), ("function", function.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index 88eb774c88..e82d4e2fda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -127,11 +127,7 @@ pub(crate) fn build_scenario_execution_plan<'py>( let model = imported(py, "dataclasses", "replace")?.call((record,), Some(&changes))?; let model_context = model_context(&project, &context, (&models, &locations, &columns))?; let result = call_with( - &imported( - py, - "sqlbuild.compiler.planner._helpers.output.plan_entry", - "plan_model", - )?, + &imported(py, "sqlbuild._native", "plan_model")?, &[ ("model", model), ("snapshot", snapshot.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs index 21f6f9a2d8..09151463ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs @@ -33,31 +33,13 @@ pub(crate) const LOG_DEBUG_EVENT: (&str, &str) = ( "sqlbuild.diagnostics.main.log_debug_event", "log_debug_event", ); -pub(crate) const CURSOR_RESOLVE_MODULE: &str = "sqlbuild.compiler.planner._helpers.resolve.cursor"; -pub(crate) const CURSOR_POLICIES: (&str, &str) = ( - "sqlbuild.compiler.planner._helpers.resolve.cursor_policies", - "resolve_start_cursor_config", -); -pub(crate) const MAXIMUM_START: (&str, &str) = ( - "sqlbuild.compiler.planner._helpers.resolve.maximum_start", - "maximum_allowed_start", -); -pub(crate) const RENDER_SOURCE_READ: (&str, &str) = ( - "sqlbuild.compiler.planner._helpers.resolve.sources", - "render_source_read_relation", -); -pub(crate) const YAML_ENTRY_LINE: (&str, &str) = ( - "sqlbuild.compiler.discovery.main._yaml_entry_line", - "yaml_entry_line", -); -pub(crate) const FRESHNESS_PLANNING: (&str, &str) = ( - "sqlbuild.compiler.source_freshness.main._planning", - "build_direct_source_freshness_planning_result", -); -pub(crate) const FRESHNESS_PROPAGATION: (&str, &str) = ( - "sqlbuild.compiler.source_freshness.main._propagation", - "build_direct_source_freshness_propagation_result", -); +pub(crate) const CURSOR_RESOLVE_MODULE: &str = "sqlbuild._native"; +pub(crate) const CURSOR_POLICIES: (&str, &str) = + ("sqlbuild._native", "resolve_start_cursor_config"); +pub(crate) const MAXIMUM_START: (&str, &str) = ("sqlbuild._native", "maximum_allowed_start"); +pub(crate) const RENDER_SOURCE_READ: (&str, &str) = + ("sqlbuild._native", "render_source_read_relation"); +pub(crate) const YAML_ENTRY_LINE: (&str, &str) = ("sqlbuild._native", "yaml_entry_line"); pub(crate) const RUN_SCOPE: (&str, &str) = ("sqlbuild.runtime.observability.main.run_scope", "run_scope"); pub(crate) const SQL_HOOK_ENTRY: (&str, &str) = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs index 0a688192c6..703a42e003 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs @@ -2,17 +2,18 @@ use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::freshness::planning::{ + build_direct_source_freshness_planning_result, build_direct_source_freshness_propagation_result, +}; use crate::bindings::_helpers::planner_phases::objects::{ model_key_names, planner_model, replace, text, tuple, }; -use crate::bindings::_helpers::planner_warehouse::constants::{ - FRESHNESS_PLANNING, FRESHNESS_PROPAGATION, RUN_SCOPE, -}; +use crate::bindings::_helpers::planner_warehouse::constants::RUN_SCOPE; use crate::bindings::_helpers::planner_warehouse::snapshot::{destination_nodes, lower}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_warehouse::main::freshness_unknown_message::freshness_unknown_message; use std::collections::BTreeSet; @@ -151,7 +152,7 @@ pub(crate) fn build_planner_source_freshness_result<'py>( &imported(py, RUN_SCOPE.0, RUN_SCOPE.1)?.call1((project.getattr("run_id")?,))?, |_| { call_with( - &imported(py, FRESHNESS_PLANNING.0, FRESHNESS_PLANNING.1)?, + wrap_pyfunction!(build_direct_source_freshness_planning_result, py)?.as_any(), &[ ("adapter", adapter.clone()), ("connection", connection.clone()), @@ -177,7 +178,7 @@ pub(crate) fn build_planner_source_freshness_result<'py>( }, )?; let propagation = call_with( - &imported(py, FRESHNESS_PROPAGATION.0, FRESHNESS_PROPAGATION.1)?, + wrap_pyfunction!(build_direct_source_freshness_propagation_result, py)?.as_any(), &[ ("source_freshness", result.clone()), ("scope", scope.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs index 42554e58ff..a3c118adf5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod document; pub(crate) mod functions; pub(crate) mod registration; +pub(crate) mod style; pub(crate) mod terminal; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs index fd3b43e9ff..df45c7b2ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs @@ -2,10 +2,11 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::presentation_text::{document, functions, terminal}; +use crate::bindings::_helpers::presentation_text::{document, functions, style, terminal}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { document::register(module)?; functions::register(module)?; + style::register(module)?; terminal::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs new file mode 100644 index 0000000000..9a6bb6d0b2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs @@ -0,0 +1,129 @@ +//! Semantic styling for human CLI output, as a Python class. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::cli_text::models::CliStyle as NativeStyle; + +/// Semantic CLI styling facade used by human-output formatters. +#[pyclass(name = "CliStyle", module = "sqlbuild._native", frozen)] +pub(crate) struct PythonCliStyle { + #[pyo3(get)] + use_color: bool, +} + +impl PythonCliStyle { + fn native(&self) -> NativeStyle { + NativeStyle { + use_color: self.use_color, + } + } +} + +#[pymethods] +impl PythonCliStyle { + #[new] + #[pyo3(signature = (*, use_color))] + fn new(use_color: &Bound<'_, PyAny>) -> PyResult { + Ok(Self { + use_color: use_color.is_truthy()?, + }) + } + + fn title(&self, text: &str) -> String { + self.native().title(text) + } + + fn section(&self, text: &str) -> String { + self.native().section(text) + } + + fn label(&self, text: &str) -> String { + self.native().label(text) + } + + fn value(&self, text: &str) -> String { + self.native().value(text) + } + + fn accent(&self, text: &str) -> String { + self.native().accent(text) + } + + fn accent_strong(&self, text: &str) -> String { + self.native().accent_strong(text) + } + + fn plan_section(&self, text: &str) -> String { + self.native().plan_section(text) + } + + fn object_name(&self, text: &str) -> String { + self.native().object_name(text) + } + + fn command(&self, text: &str) -> String { + self.native().command(text) + } + + fn muted(&self, text: &str) -> String { + self.native().muted(text) + } + + fn success(&self, text: &str) -> String { + self.native().success(text) + } + + fn success_strong(&self, text: &str) -> String { + self.native().success_strong(text) + } + + fn warning(&self, text: &str) -> String { + self.native().warning(text) + } + + fn warning_strong(&self, text: &str) -> String { + self.native().warning_strong(text) + } + + fn error(&self, text: &str) -> String { + self.native().error(text) + } + + fn error_strong(&self, text: &str) -> String { + self.native().error_strong(text) + } + + fn error_muted(&self, text: &str) -> String { + self.native().error_muted(text) + } + + fn log_label(&self, text: &str) -> String { + self.native().log_label(text) + } + + fn dbt_section(&self, text: &str) -> String { + self.native().dbt_section(text) + } + + fn dbt_label(&self, text: &str) -> String { + self.native().dbt_label(text) + } + + fn dbt_object_name(&self, text: &str) -> String { + self.native().dbt_object_name(text) + } + + fn dbt_execution_label(&self, text: &str) -> String { + self.native().dbt_execution_label(text) + } + + /// Style status text according to the status word. + #[pyo3(signature = (*, status, text = None))] + fn status(&self, status: &str, text: Option<&str>) -> String { + self.native().status_text(status, text.unwrap_or(status)) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs index 77eb7ebbfa..d626d26f8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs @@ -6,7 +6,6 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyString, PyStringMethods}; use pyo3::{PyErr, pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::assembly::analysis_session::types::Pairs; use sqlbuild_analysis::assembly::project::main::check_sql_syntax::check_sql_syntax; use sqlbuild_analysis::assembly::project::main::sql_syntax_error::sql_syntax_error as check_syntax_error; use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxFailure, SyntaxMode}; @@ -64,20 +63,6 @@ fn syntax_value_error(failure: SyntaxFailure) -> PyErr { } } -/// Whether every `(sql, placeholders)` parses, raising the `ValueError` Python's validation raises. -#[pyfunction] -fn check_native_sql_syntax( - py: Python<'_>, - request: (String, Vec<(String, Pairs)>), -) -> PyResult { - let (dialect, rows) = request; - let checks: Vec = rows - .into_iter() - .map(|(sql, placeholders)| SyntaxCheck { sql, placeholders }) - .collect(); - syntax_valid(py, &dialect, &checks) -} - /// Whether every check parses, raising what Python's validation raises. pub(crate) fn syntax_valid( py: Python<'_>, @@ -90,7 +75,6 @@ pub(crate) fn syntax_valid( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(check_native_sql_syntax, module)?)?; module.add_function(wrap_pyfunction!(sql_syntax_error, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs index 05dfc3df4a..38926e328f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs @@ -9,6 +9,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::model_config::template_data::record_reads; +use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; use crate::bindings::_helpers::render::declaration_expansion::{ DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, SpanRow, expand_declarations, span_rows, @@ -22,7 +23,6 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; -const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; const FUNCTOOLS_MODULE: &str = "functools"; const DECLARATION_FIELDS: [&str; 4] = [ "enums", @@ -40,7 +40,7 @@ fn substituted<'py>( let py: Python<'py> = sql.py(); let sources = InterpolationSources( effective_vars.downcast()?.clone(), - imported(py, UNICODE_ENVIRONMENT_MODULE, "UnicodeEnvironment")?.call0()?, + unicode_environment(py)?, if context_values.is_none() { None } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index d6fe88b2b8..1ee2b2e061 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -14,6 +14,7 @@ use crate::bindings::_helpers::render::macro_call_rows::span_row; use crate::bindings::_helpers::render::macro_expansion::{ ExpansionOutput, NativeMacroExpansion, expanded, }; +use crate::bindings::_helpers::sqltext::sql_references::extract_references; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; @@ -23,12 +24,15 @@ const SCOPES_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const SCOPES_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const REF_MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; const REF_TYPES_MODULE: &str = "sqlbuild.python_nodes.types"; -const MACRO_ARGUMENTS_MODULE: &str = - "sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments"; +const TYPED_REFERENCE_FUNCTIONS: [(&str, &str); 3] = [ + ("model", "__ref"), + ("source", "__source"), + ("seed", "__seed"), +]; const USER_MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; const ACTIVE_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; const BRIDGE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.macro_bridge.constants"; -const DIAGNOSTICS_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const DIAGNOSTICS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; const BUILTINS_MODULE: &str = "builtins"; @@ -97,12 +101,12 @@ fn generated_classes(py: Python<'_>) -> PyResult> { fn built_classes(py: Python<'_>) -> PyResult> { let native: Bound<'_, PyAny> = py.import(NATIVE_MODULE)?.into_any(); let declaration_kind: Bound<'_, PyAny> = imported(py, SCOPES_TYPES_MODULE, "DeclarationKind")?; - let ref_kinds: Bound<'_, PyAny> = - imported(py, MACRO_ARGUMENTS_MODULE, "typed_reference_kinds")?.call0()?; + let ref_kind: Bound<'_, PyAny> = imported(py, REF_TYPES_MODULE, "SqlResourceRefKind")?; + let ref_kinds: Bound<'_, PyDict> = PyDict::new(py); let ref_functions: Bound<'_, PyDict> = PyDict::new(py); - for item in ref_kinds.call_method0("items")?.try_iter()? { - let (function, kind): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; - ref_functions.set_item(kind.getattr("value")?, function)?; + for (kind, function) in TYPED_REFERENCE_FUNCTIONS { + ref_kinds.set_item(function, ref_kind.call1((kind,))?)?; + ref_functions.set_item(kind, function)?; } let store_tokens: Bound<'_, PyDict> = keywords( py, @@ -152,7 +156,7 @@ fn built_classes(py: Python<'_>) -> PyResult> { "resource_ref", imported(py, REF_MODELS_MODULE, "SqlResourceRef")?, ), - ("ref_kinds", ref_kinds), + ("ref_kinds", ref_kinds.into_any()), ("ref_functions", ref_functions.into_any()), ("store_tokens", store_tokens.into_any()), ( @@ -245,21 +249,18 @@ fn expansion_result<'py>(request: ExpansionRequest<'py>) -> PyResult = imported(py, REF_MODELS_MODULE, "SqlResourceRef")?; - let ref_kind: Bound<'py, PyAny> = imported(py, REF_TYPES_MODULE, "SqlResourceRefKind")?; - let mut references: Vec> = Vec::new(); - for (kind, name) in output.argument_references { - references.push(call_with( - &resource_ref, - &[ - ("kind", ref_kind.call1((kind,))?), - ("name", PyString::new(py, &name).into_any()), - ], - )?); - } - let argument_references: Bound<'py, PyAny> = - imported(py, MACRO_ARGUMENTS_MODULE, "call_site_sql_references")? - .call1((PyTuple::new(py, references)?,))?; + let argument_references: Bound<'py, PyAny> = if output.argument_references.is_empty() { + PyTuple::empty(py).into_any() + } else { + let calls: Vec = output + .argument_references + .iter() + .map(|(kind, name)| format!("{}(\"{name}\")", typed_reference_function(kind))) + .collect(); + let syntax: Bound<'py, PyAny> = + imported(py, SQL_ANALYSIS_MODELS_MODULE, "SqlLexicalSyntax")?.call0()?; + extract_references(&calls.join(" "), &syntax)?.into_any() + }; result_model( py, &[ @@ -272,6 +273,14 @@ fn expansion_result<'py>(request: ExpansionRequest<'py>) -> PyResult &'static str { + TYPED_REFERENCE_FUNCTIONS + .iter() + .find(|(typed, _)| *typed == kind) + .map_or("", |(_, function)| function) +} + fn optional<'py>(value: Bound<'py, PyAny>) -> Option> { (!value.is_none()).then_some(value) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs index ce6ed95913..15c2c97235 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs @@ -10,13 +10,13 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; use crate::bindings::_helpers::rules::lint_headers::{lint_body_ranges, lint_file_role}; +use crate::bindings::_helpers::rules::lint_stop::check_stop; const NATIVE: &str = "sqlbuild._native"; const MODELS: &str = "sqlbuild.lint.models"; const EXPANSION_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_ERRORS: &str = "sqlbuild.compiler.compile.exceptions"; const LINT_ERRORS: &str = "sqlbuild.lint.exceptions"; -const STOP: &str = "sqlbuild.lint.classes.stop_context"; /// Expand a single body and finish its native SQL preparation. #[pyfunction] @@ -344,7 +344,7 @@ pub(crate) fn expand_file_bodies<'py>( &[], ), )?; - imported(py, STOP, "LintStopContext")?.call_method0("check")?; + check_stop(py)?; let path = args.value("file_path")?; let role = lint_file_role( &path, @@ -459,7 +459,7 @@ pub(crate) fn prepare_expanded_files<'py>( ) -> PyResult> { let _ = dialect; let py = expanded_files.py(); - imported(py, STOP, "LintStopContext")?.call_method0("check")?; + check_stop(py)?; let files: Vec> = expanded_files.try_iter()?.collect::>()?; let mut pending: Vec> = Vec::new(); for file in &files { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs new file mode 100644 index 0000000000..2dd45f982e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs @@ -0,0 +1,84 @@ +//! The stop signal that lets a speculative lint run be abandoned between files. + +use std::sync::{Mutex, PoisonError}; + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyTuple; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{context_variable, imported}; + +const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild.lint.exceptions"; +const STOP_VARIABLE: &str = "active_lint_stop"; + +static ACTIVE_STOP: PyOnceLock> = PyOnceLock::new(); + +/// The context variable holding the active lint run's stop event, if any. +fn active_stop_variable(py: Python<'_>) -> PyResult> { + context_variable(py, &ACTIVE_STOP, STOP_VARIABLE) +} + +/// Abandon a speculative lint run on this thread once its owner no longer needs it. +pub(crate) fn check_stop(py: Python<'_>) -> PyResult<()> { + let stop: Bound<'_, PyAny> = active_stop_variable(py)?.call_method0("get")?; + if !stop.is_none() && stop.call_method0("is_set")?.is_truthy()? { + return Err(PyErr::from_value( + imported(py, LINT_EXCEPTIONS_MODULE, "LintStoppedError")?.call0()?, + )); + } + Ok(()) +} + +/// Makes `stop` the active lint stop event while entered. +#[pyclass(frozen, module = "sqlbuild._native")] +struct LintStopScope { + stop: Py, + token: Mutex>>, +} + +#[pymethods] +impl LintStopScope { + fn __enter__(&self, py: Python<'_>) -> PyResult<()> { + let token: Bound<'_, PyAny> = + active_stop_variable(py)?.call_method1("set", (self.stop.bind(py),))?; + *self.token.lock().unwrap_or_else(PoisonError::into_inner) = Some(token.unbind()); + Ok(()) + } + + #[pyo3(signature = (*_exception))] + fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult { + let token: Option> = self + .token + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + if let Some(token) = token { + active_stop_variable(py)?.call_method1("reset", (token,))?; + } + Ok(false) + } +} + +/// A scope that makes `stop` the active lint stop event while entered. +#[pyfunction] +fn lint_stop_scope(stop: &Bound<'_, PyAny>) -> LintStopScope { + LintStopScope { + stop: stop.clone().unbind(), + token: Mutex::new(None), + } +} + +/// The active lint run's stop event, or `None` outside a stop scope. +#[pyfunction] +fn active_lint_stop(py: Python<'_>) -> PyResult> { + active_stop_variable(py)?.call_method0("get") +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(lint_stop_scope, module)?)?; + module.add_function(wrap_pyfunction!(active_lint_stop, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index f13e0bc557..f87e3ba6ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -18,6 +18,7 @@ pub(crate) mod lint_expansion; pub(crate) mod lint_headers; pub(crate) mod lint_preparation; pub(crate) mod lint_sql_bodies; +pub(crate) mod lint_stop; pub(crate) mod lint_text; pub(crate) mod registration; pub(crate) mod relation_keys; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index 5abc574c40..adcb2564b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -6,8 +6,9 @@ use pyo3::{types::PyModuleMethods, wrap_pyfunction}; use crate::bindings::_helpers::rules::{ builtin_catalogue, catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, - early_sql_lint, evaluation, guidance, lint, lint_expansion, lint_sql_bodies, lint_text, - results, rule_cache, rule_plan, rule_run, rules_config, run_lint, selection, sql_rules, + early_sql_lint, evaluation, guidance, lint, lint_expansion, lint_sql_bodies, lint_stop, + lint_text, results, rule_cache, rule_plan, rule_run, rules_config, run_lint, selection, + sql_rules, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -59,6 +60,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { catalogue::register(module)?; rules_config::register(module)?; run_lint::register(module)?; + lint_stop::register(module)?; selection::register(module)?; results::register(module)?; guidance::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs index 2e32ce2c90..8c4bad037c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs @@ -17,10 +17,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; use crate::bindings::_helpers::rules::lint_expansion::build_lint_expansion_context; +use crate::bindings::_helpers::rules::lint_stop::check_stop; const NATIVE_MODULE: &str = "sqlbuild._native"; const EXPANSION_MODULE: &str = "sqlbuild._native"; -const STOP_CONTEXT_MODULE: &str = "sqlbuild.lint.classes.stop_context"; const LINT_MODELS_MODULE: &str = "sqlbuild.lint.models"; fn resolved(path: &Path) -> PathBuf { @@ -84,12 +84,6 @@ fn collect_lint_project_files<'py>( ) } -/// `LintStopContext.check()`: abandon a speculative run its owner no longer needs. -fn check_stop(py: Python<'_>) -> PyResult<()> { - imported(py, STOP_CONTEXT_MODULE, "LintStopContext")?.call_method0("check")?; - Ok(()) -} - /// The keyword arguments of one lint run. struct LintRun<'py> { project_dir: Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs index ec680f1ab9..ec8592a9ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -22,7 +22,7 @@ use crate::bindings::_helpers::compiled_project::model_loop_named::scope_path_me const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; -const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const COLLECTOR_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const INACCESSIBLE_DECLARATION_CODE: &str = "S006"; const SCHEMA_USAGE_KINDS: [&str; 2] = ["schema", "enum"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs index 40b2b87e1c..76dcf00193 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const COLLECTOR_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const ERROR_SEVERITY: &str = "error"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index 37bee60bec..372315ecd9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -11,6 +11,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::discovery::scope_snapshot::discover_scope_snapshot; +use crate::bindings::_helpers::frontier::cache_directory::cache_directory; use crate::bindings::_helpers::scopes::scope_cache::{ project_fingerprint, read_scope_index_cache, write_scope_index_cache, }; @@ -19,7 +20,6 @@ use crate::bindings::_helpers::scopes::scope_tolerant::scope_tolerant_index; use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; const NATIVE_MODULE: &str = "sqlbuild._native"; -const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; const SCOPE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.scopes.constants"; const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; @@ -33,14 +33,11 @@ const MACRO_INVENTORY_MODULE: &str = "sqlbuild.compiler.compile._helpers.render. /// Where the scope index cache file of a project lives: `(directory, file name)`. fn cache_location(project_dir: &Bound<'_, PyAny>) -> PyResult<(PathBuf, String)> { let py: Python<'_> = project_dir.py(); - let directory: Bound<'_, PyAny> = - imported(py, CACHE_DIRECTORY_MODULE, "compiler_cache_directory")? - .call1((project_dir,))? - .div(imported( - py, - SCOPE_CONSTANTS_MODULE, - "SCOPE_CACHE_DIRECTORY_NAME", - )?)?; + let directory: Bound<'_, PyAny> = cache_directory(project_dir)?.div(imported( + py, + SCOPE_CONSTANTS_MODULE, + "SCOPE_CACHE_DIRECTORY_NAME", + )?)?; Ok(( directory.extract()?, imported(py, SCOPE_CONSTANTS_MODULE, "SCOPE_CACHE_FILENAME")?.extract()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 729601ba64..2b26d2a393 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -15,6 +15,7 @@ use sqlbuild_core::seed_command::main::seed_exit_code::seed_exit_code; use sqlbuild_core::seed_command::models::SeedRow; use crate::bindings::_helpers::command_runtime::invocation::resolve_command_invocation; +use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -22,9 +23,8 @@ const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; -const SEED_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._seed"; -const SEED_STATE_MODULE: &str = "sqlbuild.compiler.planner.main.commands._seed_state"; -const OPEN_CONNECTION_MODULE: &str = "sqlbuild.runtime.contracts.main.open_connection"; +const SEED_PLAN_MODULE: &str = "sqlbuild._native"; +const SEED_STATE_MODULE: &str = "sqlbuild._native"; const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const SUCCESS_STATUS: &str = "success"; @@ -248,14 +248,8 @@ fn compile_seeds<'py>( ("on_connection_error", relay.getattr("on_connection_error")?), ], )?; - let connection: Bound<'py, PyAny> = call_with( - &imported(py, OPEN_CONNECTION_MODULE, "open_connection_with_hooks")?, - &[ - ("adapter", adapter.clone()), - ("connection_config", connection_config), - ("hooks", hooks), - ], - )?; + let connection: Bound<'py, PyAny> = + connection_with_hooks(&adapter, &connection_config, &hooks)?; let read: PyResult> = call_with( &imported(py, SEED_STATE_MODULE, "read_selected_seed_fingerprints")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 018dbf9435..fcc0b9a8e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -36,7 +36,7 @@ const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; -const TEST_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._sql_test"; +const TEST_PLAN_MODULE: &str = "sqlbuild._native"; const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; const EXECUTOR_PIPELINE_MODELS: &str = "sqlbuild.executor.pipeline.models"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs index a90e0ead0e..dabf3065ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs @@ -1,8 +1,39 @@ //! Project native planning failures into existing located diagnostic exceptions. -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::sql_test_glue::plan_values::string; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::sql_test_attachment::main::sql_test_cte_location::sql_test_cte_location; +use sqlbuild_analysis::sql_test_attachment::models::CteLocation; + +const SOURCE_LOCATION_MODULE: &str = "sqlbuild.spec.contracts.models"; + +/// The location of `call` inside one CTE of a SQL test block, else the CTE header, else the block. +fn cte_call_location<'py>( + py: Python<'py>, + test: &Bound<'py, PyAny>, + (cte, call): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let test_file: Bound<'py, PyAny> = test.getattr("test_file")?; + let call: Option = call.extract()?; + let located: CteLocation = sql_test_cte_location( + &test_file.getattr("contents")?.extract::()?, + &test + .getattr("test_block")? + .getattr("sql_body")? + .extract::()?, + &cte.extract::()?, + call.as_deref(), + ); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("path", test_file.getattr("relative_path")?)?; + arguments.set_item("line", located.line)?; + arguments.set_item("column", located.column)?; + arguments.set_item("end_line", located.end_line)?; + arguments.set_item("end_column", located.end_column)?; + imported(py, SOURCE_LOCATION_MODULE, "SourceLocation")?.call((), Some(&arguments)) +} pub(super) fn planning_error( py: Python<'_>, @@ -38,27 +69,10 @@ pub(super) fn planning_error( "Mock it in the test, for example {} AS (SELECT ...), or call __ref(\"\") on a model the test runs.", payload.get_item("mockCte")?.extract::()? ); - let kwargs = pyo3::types::PyDict::new(py); - use pyo3::types::PyDictMethods; + let kwargs = PyDict::new(py); kwargs.set_item("help", help)?; let class = if let Some(test) = found { - kwargs.set_item( - "location", - call_with( - &imported( - py, - "sqlbuild.compiler.compile.classes.sql_test_cte_locator", - "SqlTestCteLocator", - )? - .getattr("locate")?, - &[ - ("test_file", test.getattr("test_file")?), - ("test_block", test.getattr("test_block")?), - ("cte_name", cte), - ("call", call), - ], - )?, - )?; + kwargs.set_item("location", cte_call_location(py, &test, (&cte, &call))?)?; "SqlTestReferenceError" } else { kwargs.set_item( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs index 99fad543d4..8fd76caa2a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs @@ -19,7 +19,7 @@ const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; -const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const COLLECTOR_MODULE: &str = "sqlbuild._native"; const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; /// `(kind, name, package, call_argument_count)` for one reference. @@ -176,7 +176,10 @@ fn scan_sql_reference_calls<'py>( /// Logical SQL refs outside comments and quotes, reporting rejected calls as P012. #[pyfunction(name = "extract_sql_references")] #[pyo3(signature = (*, sql, syntax))] -fn extract_references<'py>(sql: &str, syntax: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn extract_references<'py>( + sql: &str, + syntax: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = syntax.py(); let (references, invalid_calls) = scanned(py, sql, syntax)?; let report: Bound<'py, PyAny> = imported(py, COLLECTOR_MODULE, "report_compile_diagnostic")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs index a2377ff690..9b30e3252c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs @@ -23,7 +23,7 @@ use crate::bindings::_helpers::state_rows::timestamps::{ date_time_typed, in_utc, offset_aware, presumed_utc, }; -const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.source_freshness.exceptions"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const TYPES_MODULE: &str = "sqlbuild.compiler.source_freshness.types"; const TIMESTAMP_KIND: &str = "timestamp"; const INTEGER_KIND: &str = "integer"; @@ -49,7 +49,7 @@ fn python_error(py: Python<'_>, error: FreshnessError) -> PyErr { /// Hash the stable source freshness identity used as a graph input. #[pyfunction(name = "source_freshness_data_version_hash")] #[pyo3(signature = (*, source_name, strategy, value_kind, data_version))] -fn version_hash( +pub(crate) fn version_hash( source_name: &str, strategy: &str, value_kind: &str, @@ -66,7 +66,7 @@ fn observation_error(py: Python<'_>, message: &str) -> PyErr { /// A source freshness value normalized for stable state storage. #[pyfunction(name = "normalize_source_freshness_data_version")] #[pyo3(signature = (*, value, value_kind))] -fn normalize<'py>( +pub(crate) fn normalize<'py>( value: &Bound<'py, PyAny>, value_kind: &Bound<'py, PyAny>, ) -> PyResult> { @@ -163,7 +163,7 @@ fn comparable(record: &Bound<'_, PyAny>) -> PyResult { /// Whether two source freshness records are equivalent for skip decisions. #[pyfunction(name = "source_freshness_records_equivalent")] #[pyo3(signature = (*, previous_record, current_record, lag_tolerance = None))] -fn equivalent( +pub(crate) fn equivalent( previous_record: &Bound<'_, PyAny>, current_record: &Bound<'_, PyAny>, lag_tolerance: Option<&str>, @@ -197,7 +197,7 @@ fn aware_utc(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult( +pub(crate) fn age_policy<'py>( policy: &Bound<'py, PyAny>, data_version: &Bound<'py, PyAny>, observed_at: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 46f3b76be5..1f4e40a899 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -29,6 +29,7 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { analysis::sql_tests::register(module)?; sqltext::registration::register(module)?; boundary::oracles::register(module)?; + boundary::expected_errors::register(module)?; discovery::registration::register(module)?; scopes::registration::register(module)?; model_config::registration::register(module)?; @@ -52,6 +53,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { cursor::registration::register(module)?; state_rows::registration::register(module)?; diagnostics::redaction::register(module)?; + diagnostics::trackers::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-state/src/freshness/constants.rs b/crates/sqlbuild-state/src/freshness/constants.rs index 280fa1be44..19e375d039 100644 --- a/crates/sqlbuild-state/src/freshness/constants.rs +++ b/crates/sqlbuild-state/src/freshness/constants.rs @@ -28,3 +28,17 @@ pub(crate) const MARCH_BASED_MONTH_DAYS: i128 = 153; pub(crate) const MONTH_LENGTH_DIVISOR: i128 = 5; pub(crate) const MONTHS_FROM_MARCH: i128 = 9; pub(crate) const FEBRUARY: u32 = 2; +pub const SOURCE_FRESHNESS_TABLE_NAME: &str = "_sqlbuild_source_freshness"; +/// Columns a source freshness state table must have, in Python's declaration order. +pub const REQUIRED_STATE_COLUMNS: [&str; 6] = [ + "source_name", + "run_id", + "strategy", + "value_kind", + "data_version_hash", + "observed_at", +]; +pub(crate) const STATE_SELECT_COLUMNS: &str = "source_name, target_database, target_schema, \ +target_name, run_id, strategy, value_kind, data_version, data_version_hash, observed_at"; +pub(crate) const PHYSICAL_TABLE_SOURCE_FRAGMENT: &str = "requires a physical table source"; +pub(crate) const INCOMPLETE_CONFIGURATION_FRAGMENT: &str = "incomplete"; diff --git a/crates/sqlbuild-state/src/freshness/main/configuration_error.rs b/crates/sqlbuild-state/src/freshness/main/configuration_error.rs new file mode 100644 index 0000000000..8d25018f4a --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/configuration_error.rs @@ -0,0 +1,11 @@ +//! `configuration_error`: whether an observation failure is a source configuration error. + +use crate::freshness::constants::{ + INCOMPLETE_CONFIGURATION_FRAGMENT, PHYSICAL_TABLE_SOURCE_FRAGMENT, +}; + +/// Configuration errors fail planning; every other observation failure marks the source unknown. +pub fn configuration_error(message: &str) -> bool { + message.contains(PHYSICAL_TABLE_SOURCE_FRAGMENT) + || message.contains(INCOMPLETE_CONFIGURATION_FRAGMENT) +} diff --git a/crates/sqlbuild-state/src/freshness/main/mod.rs b/crates/sqlbuild-state/src/freshness/main/mod.rs index 6e7cddd828..9ec47ed663 100644 --- a/crates/sqlbuild-state/src/freshness/main/mod.rs +++ b/crates/sqlbuild-state/src/freshness/main/mod.rs @@ -1,4 +1,8 @@ pub mod age_status; +pub mod configuration_error; pub mod data_version_hash; pub mod normalized_data_version; +pub mod read_latest_sql; pub mod records_equivalent; +pub mod relevant_sources_sql; +pub mod single_value_problem; diff --git a/crates/sqlbuild-state/src/freshness/main/read_latest_sql.rs b/crates/sqlbuild-state/src/freshness/main/read_latest_sql.rs new file mode 100644 index 0000000000..9dc677915d --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/read_latest_sql.rs @@ -0,0 +1,13 @@ +//! `read_latest_sql`: the windowed SELECT of the latest state row per source identity. + +use crate::freshness::constants::STATE_SELECT_COLUMNS; + +/// Latest row per source, target database, schema and name, newest observation and run first. +pub fn read_latest_sql(qualified_name: &str) -> String { + format!( + "SELECT {STATE_SELECT_COLUMNS} FROM (SELECT {STATE_SELECT_COLUMNS}, ROW_NUMBER() OVER (\ +PARTITION BY source_name, target_database, target_schema, target_name \ +ORDER BY observed_at DESC, run_id DESC) AS __sqlbuild_latest_rank FROM {qualified_name}) \ +AS __sqlbuild_latest_source_freshness WHERE __sqlbuild_latest_rank = 1" + ) +} diff --git a/crates/sqlbuild-state/src/freshness/main/relevant_sources_sql.rs b/crates/sqlbuild-state/src/freshness/main/relevant_sources_sql.rs new file mode 100644 index 0000000000..99b2be1e49 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/relevant_sources_sql.rs @@ -0,0 +1,13 @@ +//! `relevant_sources_sql`: a latest-state read narrowed to the named sources. + +/// `read_sql` filtered to `source_names`, each quoted as a SQL string literal. +pub fn relevant_sources_sql(read_sql: &str, source_names: &[String]) -> String { + let literals: Vec = source_names + .iter() + .map(|name| format!("'{}'", name.replace('\'', "''"))) + .collect(); + format!( + "SELECT * FROM ({read_sql}) AS __sqlbuild_relevant WHERE source_name IN ({})", + literals.join(", ") + ) +} diff --git a/crates/sqlbuild-state/src/freshness/main/single_value_problem.rs b/crates/sqlbuild-state/src/freshness/main/single_value_problem.rs new file mode 100644 index 0000000000..7f972a450b --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/single_value_problem.rs @@ -0,0 +1,20 @@ +//! `single_value_problem`: why a freshness query result is not one non-null value. + +use crate::freshness::models::QueryShape; + +/// The observation error message for a result that is not exactly one non-null value. +pub fn single_value_problem(source_name: &str, shape: QueryShape) -> Option { + if shape.columns != 1 { + return Some(format!( + "source '{source_name}' freshness query must return exactly one column" + )); + } + if shape.rows != 1 { + return Some(format!( + "source '{source_name}' freshness query must return exactly one row" + )); + } + shape + .first_value_null + .then(|| format!("source '{source_name}' freshness data_version cannot be null")) +} diff --git a/crates/sqlbuild-state/src/freshness/models.rs b/crates/sqlbuild-state/src/freshness/models.rs index 99ca646dfe..dbe1341a71 100644 --- a/crates/sqlbuild-state/src/freshness/models.rs +++ b/crates/sqlbuild-state/src/freshness/models.rs @@ -30,3 +30,11 @@ pub enum AgeStatus { Warn, Error, } + +/// The shape of a single-value freshness query result. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct QueryShape { + pub columns: usize, + pub rows: usize, + pub first_value_null: bool, +} diff --git a/crates/sqlbuild-state/src/freshness/tests.rs b/crates/sqlbuild-state/src/freshness/tests.rs index ead02b3221..8ac14ee7fc 100644 --- a/crates/sqlbuild-state/src/freshness/tests.rs +++ b/crates/sqlbuild-state/src/freshness/tests.rs @@ -2,5 +2,7 @@ mod equivalence; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_observation.rs"] +mod observation; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-state/src/freshness/tests/test_observation.rs b/crates/sqlbuild-state/src/freshness/tests/test_observation.rs new file mode 100644 index 0000000000..77274157bb --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/tests/test_observation.rs @@ -0,0 +1,143 @@ +use crate::freshness::main::configuration_error::configuration_error; +use crate::freshness::main::read_latest_sql::read_latest_sql; +use crate::freshness::main::relevant_sources_sql::relevant_sources_sql; +use crate::freshness::main::single_value_problem::single_value_problem; +use crate::freshness::models::QueryShape; +use crate::freshness::tests::test_types::{ + ConfigurationErrorTestCase, ReadLatestTestCase, RelevantSourcesTestCase, SingleValueTestCase, +}; + +#[test] +fn given_qualified_name_when_reading_latest_then_ranks_newest_row_per_identity() { + let test_cases = [ReadLatestTestCase { + description: "one schema-qualified state table", + qualified_name: "main.state", + expected_sql: "SELECT source_name, target_database, target_schema, target_name, run_id, \ +strategy, value_kind, data_version, data_version_hash, observed_at FROM (SELECT source_name, \ +target_database, target_schema, target_name, run_id, strategy, value_kind, data_version, \ +data_version_hash, observed_at, ROW_NUMBER() OVER (PARTITION BY source_name, target_database, \ +target_schema, target_name ORDER BY observed_at DESC, run_id DESC) AS __sqlbuild_latest_rank \ +FROM main.state) AS __sqlbuild_latest_source_freshness WHERE __sqlbuild_latest_rank = 1", + }]; + + for test_case in test_cases { + assert_eq!( + read_latest_sql(test_case.qualified_name), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_source_names_when_narrowing_latest_read_then_quotes_each_literal() { + let test_cases = [ + RelevantSourcesTestCase { + description: "one plain name", + source_names: &["orders"], + expected_sql: "SELECT * FROM (R) AS __sqlbuild_relevant WHERE source_name IN ('orders')", + }, + RelevantSourcesTestCase { + description: "quotes are doubled", + source_names: &["a", "o'b"], + expected_sql: "SELECT * FROM (R) AS __sqlbuild_relevant WHERE source_name IN ('a', 'o''b')", + }, + ]; + + for test_case in test_cases { + let names: Vec = test_case + .source_names + .iter() + .map(|name| (*name).to_owned()) + .collect(); + + assert_eq!( + relevant_sources_sql("R", &names), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_query_shape_when_checking_single_value_then_reports_first_problem() { + let test_cases = [ + SingleValueTestCase { + description: "two columns win over two rows", + shape: QueryShape { + columns: 2, + rows: 2, + first_value_null: false, + }, + expected_problem: Some("source 's' freshness query must return exactly one column"), + }, + SingleValueTestCase { + description: "no rows", + shape: QueryShape { + columns: 1, + rows: 0, + first_value_null: false, + }, + expected_problem: Some("source 's' freshness query must return exactly one row"), + }, + SingleValueTestCase { + description: "a null value", + shape: QueryShape { + columns: 1, + rows: 1, + first_value_null: true, + }, + expected_problem: Some("source 's' freshness data_version cannot be null"), + }, + SingleValueTestCase { + description: "one value", + shape: QueryShape { + columns: 1, + rows: 1, + first_value_null: false, + }, + expected_problem: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + single_value_problem("s", test_case.shape).as_deref(), + test_case.expected_problem, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_observation_message_when_classifying_then_configuration_errors_are_detected() { + let test_cases = [ + ConfigurationErrorTestCase { + description: "physical table requirement", + message: "source 's' adapter freshness requires a physical table source", + expected_configuration_error: true, + }, + ConfigurationErrorTestCase { + description: "incomplete configuration", + message: "source 's' has incomplete sql freshness configuration", + expected_configuration_error: true, + }, + ConfigurationErrorTestCase { + description: "query failure", + message: "source 's' freshness query failed: boom", + expected_configuration_error: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + configuration_error(test_case.message), + test_case.expected_configuration_error, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/freshness/tests/test_types.rs b/crates/sqlbuild-state/src/freshness/tests/test_types.rs index 0632d8083a..ec118da065 100644 --- a/crates/sqlbuild-state/src/freshness/tests/test_types.rs +++ b/crates/sqlbuild-state/src/freshness/tests/test_types.rs @@ -10,3 +10,27 @@ pub(crate) struct EquivalenceTestCase { pub(crate) lag_tolerance: Option<&'static str>, pub(crate) expected_equivalent: Result, } + +pub(crate) struct RelevantSourcesTestCase { + pub(crate) description: &'static str, + pub(crate) source_names: &'static [&'static str], + pub(crate) expected_sql: &'static str, +} + +pub(crate) struct SingleValueTestCase { + pub(crate) description: &'static str, + pub(crate) shape: crate::freshness::models::QueryShape, + pub(crate) expected_problem: Option<&'static str>, +} + +pub(crate) struct ConfigurationErrorTestCase { + pub(crate) description: &'static str, + pub(crate) message: &'static str, + pub(crate) expected_configuration_error: bool, +} + +pub(crate) struct ReadLatestTestCase { + pub(crate) description: &'static str, + pub(crate) qualified_name: &'static str, + pub(crate) expected_sql: &'static str, +} diff --git a/fensu.toml b/fensu.toml index c3c999eea3..1738998284 100644 --- a/fensu.toml +++ b/fensu.toml @@ -58,6 +58,51 @@ rule = "FFR309" path = "src/sqlbuild/spec/contracts/__init__.py" reason = "Native bindings own configuration resolution; this package only declares the configuration models, constants and errors callers import." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/planner/__init__.py" +reason = "Native bindings own planning, plan entries, plan output and command plans; this package only declares the plan models, types, constants, errors and the fixture inference cache the native planner reads." + +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/source_freshness/__init__.py" +reason = "Native bindings own source freshness observation, planning, propagation and state reads and writes; this package only declares the models, types and constants they build." + +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/cost/constants.py" +reason = "Native bindings own run-cost allocation, Snowflake cost collection, the statement ledger and the cost context; this package only declares the cost models, types and constants they use." + +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/project_assembly/__init__.py" +reason = "Native bindings own project assembly and SQL syntax checks; this package only declares the resource fact models the native assembly builds." + +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/contract_adoption/__init__.py" +reason = "Native bindings own contract comparison, edits and repository writes; this package only declares the adoption models, types and errors the native command builds and raises." + +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/pipeline/__init__.py" +reason = "Native bindings own the compile-and-plan pipeline; this package declares the pipeline models and keeps the custom materialization loader that imports project code." + +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/runtime/contracts/__init__.py" +reason = "Native bindings open planning connections and report their lifecycle; this package only declares the connection hook models and types callers pass." + +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/errors/contracts/__init__.py" +reason = "Native bindings read expected-error codes, messages and help; this package only declares the shared and executor input errors callers raise." + +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/presentation/__init__.py" +reason = "Native bindings own CLI styling, terminal width and the transient line coordinator; this package only declares the display options and presentation types callers pass." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/dag/types.py" diff --git a/python-boundary.toml b/python-boundary.toml index 924fcc11ea..ddce5dfd34 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 21360 +remaining_lines = 13066 [[stays]] path = "executor" diff --git a/scripts/cold_compile_performance/_helpers/varied_benchmark.py b/scripts/cold_compile_performance/_helpers/varied_benchmark.py index 7fe7dd708c..651959d413 100644 --- a/scripts/cold_compile_performance/_helpers/varied_benchmark.py +++ b/scripts/cold_compile_performance/_helpers/varied_benchmark.py @@ -11,6 +11,7 @@ from sqlbuild.rule_engine.models import Rule, RulesConfig from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import VariedCompileCacheTestCase from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + _RETAINED_BENCHMARK_PAYLOAD_KEYS, FreshProcessCompileBenchmarkResult, _run_fresh_process_compile_benchmark, ) @@ -23,7 +24,10 @@ def run_varied_cache_benchmark( _assert_complete_rules(project_dir) measurements: dict[str, FreshProcessCompileBenchmarkResult] = {} measurements["cold"] = _measure( - project_dir=project_dir, label="cold", budget=test_case.expected_cold_max_seconds + project_dir=project_dir, + label="cold", + budget=test_case.expected_cold_max_seconds, + retained_payload_keys=(*_RETAINED_BENCHMARK_PAYLOAD_KEYS, "resources"), ) measurements["warm"] = _measure( project_dir=project_dir, label="warm", budget=test_case.expected_warm_max_seconds @@ -109,10 +113,17 @@ def _measure_edit( return measurements -def _measure(*, project_dir: Path, label: str, budget: float) -> FreshProcessCompileBenchmarkResult: +def _measure( + *, + project_dir: Path, + label: str, + budget: float, + retained_payload_keys: tuple[str, ...] = _RETAINED_BENCHMARK_PAYLOAD_KEYS, +) -> FreshProcessCompileBenchmarkResult: return _run_fresh_process_compile_benchmark( project_dir=project_dir, label=f"varied-{label}", expected_max_wall_seconds=budget, compile_args=(), + retained_payload_keys=retained_payload_keys, ) diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 9207da6f01..d05df9c59f 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -658,7 +658,6 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: ), } ANALYSIS_CODE_SCAN_ROOTS: tuple[str, ...] = ( - "compile/_helpers/analysis", "compile/_helpers/diagnostics", "contracts", "sql_analysis", diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index e528e91df7..950db72806 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -37,7 +37,6 @@ ALLOWED_METADATA_LOOP_PATHS: frozenset[str] = frozenset( { "src/sqlbuild/adapter/relations/main/relation_lookup.py", - "src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py", "src/sqlbuild/compiler/planner/_helpers/warehouse/semantic_sources.py", "src/sqlbuild/executor/janitor/_helpers/plan.py", "src/sqlbuild/executor/pipeline/_helpers/testing.py", @@ -187,11 +186,6 @@ PROVIDER_CLASS_NAME: str = "Provider" PROVIDER_MODULE_PARTS: tuple[str, ...] = ("src", "sqlbuild", "providers.py") PUBLIC_EXPORT_ASSIGNMENT_TARGETS: tuple[str, ...] = ("__all__",) -PUBLIC_COLOR_ENTRY_PARTS: frozenset[tuple[str, ...]] = frozenset( - { - ("src", "sqlbuild", "presentation", "main", "terminal_columns.py"), - } -) ROOT_SCOPE_NAME: str = "root" RUNTIME_ROOT_PARTS: tuple[str, ...] = ("src", "sqlbuild") SELECTOR_MARKER: str = "+" diff --git a/scripts/fensu_policy/rules/canonical_operations.py b/scripts/fensu_policy/rules/canonical_operations.py index 8968f85d6d..de2907a050 100644 --- a/scripts/fensu_policy/rules/canonical_operations.py +++ b/scripts/fensu_policy/rules/canonical_operations.py @@ -19,38 +19,12 @@ NAME_REFERENCE_KIND, PLANNER_PATH_PREFIX, POLICY_EVALUATION_SCOPES, - PUBLIC_COLOR_ENTRY_PARTS, ROOT_SCOPE_NAME, SOURCE_FRESHNESS_MARKERS, SOURCE_FRESHNESS_SINGULAR_WRITER, ) -@rule( - code="XSB041", - family=Family.CUSTOM, - slug="color-capability-entry", - message="terminal capability imports must use the presentation main entry", - remediation="Import terminal_columns from sqlbuild.presentation.main.terminal_columns.", -) -def color_capability_entry(*, module: object, ctx: RuleContext) -> list[Fault]: - del module - if ctx.repo_relative_parts() in PUBLIC_COLOR_ENTRY_PARTS: - return [] - raw_module_parts: tuple[str, ...] = tuple( - "sqlbuild.presentation._helpers.terminal_capabilities".split(".") - ) - faults: list[Fault] = [] - imported: ImportFact - for imported in ctx.facts.references().imports: - imports_raw_module: bool = imported.module_parts == raw_module_parts or any( - alias.imported_parts == raw_module_parts for alias in imported.aliases - ) - if imports_raw_module: - faults.append(ctx.fault_at(location=imported.location)) - return faults - - @rule( code="XSB052", family=Family.CUSTOM, diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index e3ca62302f..2bc52aa41f 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,10 +1,14 @@ """Private SQLBuild native engine bindings.""" -from collections.abc import Callable, Iterable, Mapping, Sequence, Set +import threading +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence, Set +from concurrent.futures import Future from contextlib import AbstractContextManager from contextvars import Token from datetime import date, datetime +from decimal import Decimal from pathlib import Path, PurePath +from threading import Event from typing import Any, Literal, TextIO, TypedDict from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -52,7 +56,7 @@ from sqlbuild.cli.commands.models import ( ) from sqlbuild.cli.commands.types import CliCommand, CompileLineageMode, ScopeIndexLoader from sqlbuild.cli.compile.models import CompileAnalysis -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt, CompileReuseRequest +from sqlbuild.cli.compile_reuse.models import CompileReuseRequest from sqlbuild.cli.entry.models import ( AuthoredTargetWarehouse, CommandWarehouseScope, @@ -74,8 +78,8 @@ from sqlbuild.compiler.auditing.types import ( AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.models import ( + AnalysisCacheContext, CompileAdapterContext, CompileAnalysisSelection, CompileAuditInput, @@ -211,6 +215,7 @@ from sqlbuild.compiler.planner.models import ( AuditPlanEntry, BackfillResult, ChangeDetectionResult, + CloneSourcePlanEntry, ColumnMigrationPlanEntry, ColumnMigrationPlanning, ColumnRenameHint, @@ -224,6 +229,7 @@ from sqlbuild.compiler.planner.models import ( DirectModelVersionIdentities, EmptyCursorInputDecision, FixtureRelationMetadata, + FunctionPlanEntry, FutureCursorSafetyEvidence, MaximumStartPolicyInputs, MaximumStartSafetyEvidence, @@ -251,6 +257,7 @@ from sqlbuild.compiler.planner.models import ( PlannerChangeResults, PlannerEntryResults, PlannerIdentityContext, + PlannerModelEntryResults, PlannerOverrides, PlannerPolicies, PlannerRelationsContext, @@ -262,6 +269,7 @@ from sqlbuild.compiler.planner.models import ( PlannerSelection, PlannerWarehouseState, PlanOutput, + PlanOutputExtras, PlanWarning, QueryShape, ReferenceRename, @@ -337,9 +345,14 @@ from sqlbuild.compiler.scopes.models import ( ) from sqlbuild.compiler.scopes.types import DeclarationKind, JsonValue, ScopeQueryTarget from sqlbuild.compiler.source_freshness.models import ( + AdapterSourceFreshnessBatch, DirectSourceFreshnessPlanningResult, + DirectSourceFreshnessPropagationResult, SourceFreshnessIdentity, + SourceFreshnessObservation, SourceFreshnessRecord, + SourceFreshnessRenderers, + SourceFreshnessSet, ) from sqlbuild.compiler.source_freshness.types import ( SourceFreshnessAgeStatus, @@ -357,7 +370,14 @@ from sqlbuild.compiler.sql_test_glue.models import ( NativeSqlTestPlanningRequest, ) from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestPlanRow -from sqlbuild.cost.models import CostRunRecord, StatementExecutionTelemetry +from sqlbuild.cost.models import ( + CostResourceContext, + CostRunRecord, + QueryCostObservation, + RunCostSummary, + StatementExecutionTelemetry, + StatementLedgerEntry, +) from sqlbuild.cursor_algebra.models import ( AlignedInterval, DateValue, @@ -365,6 +385,7 @@ from sqlbuild.cursor_algebra.models import ( TimestampValue, ) from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar +from sqlbuild.diagnostics.models import PartialBuildPhaseTimings, ProcessResourceUsage from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import ( BuildExecutionResult, @@ -422,9 +443,8 @@ from sqlbuild.lint.models import ( RuleFixResult, ) from sqlbuild.lint.types import RelationKeyRow, RelationKeys -from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.models import DisplayOptions -from sqlbuild.presentation.types import CompletionState +from sqlbuild.presentation.types import CompletionState, TransientLineOwner from sqlbuild.provider.classes.container import ProviderContainer from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.rule_engine.models import ( @@ -1636,6 +1656,36 @@ class NativeReuseAttempt: def check_compile_reuse(check: dict[str, object]) -> NativeReuseAttempt: ... def run_compile(request: CompileCommandRequest) -> int: ... + +class CompileReuseAttempt: + """Result of checking for a reusable compile, carried into a full compile on a miss.""" + + outcome: str + project_dir: Path + started: float + check_ms: int + exit_code: int | None + entry_path: Path | None + native: NativeReuseAttempt | None + selected_target: str | None + json_output: bool + def __init__( + self, + *, + outcome: str, + project_dir: Path, + started: float, + check_ms: int = 0, + exit_code: int | None = None, + entry_path: Path | None = None, + native: NativeReuseAttempt | None = None, + selected_target: str | None = None, + json_output: bool = False, + ) -> None: ... + +COMPILE_REUSE_RACY_WINDOW_NS: int +COMPILE_REUSE_EXCLUDED_ROOT_DIRECTORIES: frozenset[str] + def attempt_compile_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: ... class CompileArtifactWrites: @@ -2941,12 +2991,8 @@ class DiscoveredSqlModelFile: @property def extract_implicit_alias_columns(self) -> bool: ... -def parse_schema_document( - loaded: object, path: str, classes: dict[str, object] -) -> tuple[SchemaSeedEntry, ...]: ... -def parse_sources_document( - loaded: object, path: str, classes: dict[str, object] -) -> tuple[SourceEntry, ...]: ... +def parse_schema_document(loaded: object, path: str, /) -> tuple[SchemaSeedEntry, ...]: ... +def parse_sources_document(loaded: object, path: str, /) -> tuple[SourceEntry, ...]: ... def analyze_compile_project( *, project_dir: Path, @@ -3452,7 +3498,114 @@ def effective_config_connection( def effective_config_settings( settings: tuple[object, object], overrides: frozenset[str], names: list[str], / ) -> dict[str, object]: ... -def yaml_entry_line_index(contents: str, /) -> list[tuple[str, int]]: ... +def yaml_entry_line(*, contents: str, name: str) -> int | None: ... + +class CollectedCompileDiagnostics: + def __enter__(self) -> CollectedCompileDiagnostics: ... + def __exit__(self, *exception: object) -> bool: ... + def add(self, *, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None: ... + @property + def diagnostics(self) -> tuple[CompilerDiagnostic, ...]: ... + +class CliStyle: + use_color: bool + def __init__(self, *, use_color: bool) -> None: ... + def title(self, text: str) -> str: ... + def section(self, text: str) -> str: ... + def label(self, text: str) -> str: ... + def value(self, text: str) -> str: ... + def accent(self, text: str) -> str: ... + def accent_strong(self, text: str) -> str: ... + def plan_section(self, text: str) -> str: ... + def object_name(self, text: str) -> str: ... + def command(self, text: str) -> str: ... + def muted(self, text: str) -> str: ... + def success(self, text: str) -> str: ... + def success_strong(self, text: str) -> str: ... + def warning(self, text: str) -> str: ... + def warning_strong(self, text: str) -> str: ... + def error(self, text: str) -> str: ... + def error_strong(self, text: str) -> str: ... + def error_muted(self, text: str) -> str: ... + def log_label(self, text: str) -> str: ... + def dbt_section(self, text: str) -> str: ... + def dbt_label(self, text: str) -> str: ... + def dbt_object_name(self, text: str) -> str: ... + def dbt_execution_label(self, text: str) -> str: ... + def status(self, *, status: str, text: str | None = None) -> str: ... + +def collect_compile_diagnostics() -> CollectedCompileDiagnostics: ... + +class TransientLineCoordinator: + lock: threading.RLock + def __init__(self) -> None: ... + def claim(self, *, stream: TextIO, owner: TransientLineOwner) -> None: ... + def release(self, *, owner: TransientLineOwner) -> None: ... + def clear_active(self) -> None: ... + def write_persistent(self, *, stream: TextIO, text: str) -> None: ... + +def shared_transient_line_coordinator() -> TransientLineCoordinator: ... + +class BuildPhaseTimingTracker: + compile_seconds: float | None + planning_seconds: float | None + connection_preparation_seconds: float | None + schema_preparation_seconds: float | None + execution_seconds: float | None + cost_collection_seconds: float | None + def __init__(self, *, monotonic: Callable[[], float] | None = None) -> None: ... + def snapshot(self) -> PartialBuildPhaseTimings: ... + @classmethod + def current(cls) -> BuildPhaseTimingTracker | None: ... + def scope(self) -> PhaseTimingScope: ... + +class PhaseTimingScope: + def __enter__(self) -> None: ... + def __exit__(self, *exception: object) -> bool: ... + +class ProcessResourceTracker: + def __init__( + self, + *, + monotonic: Callable[[], float] | None = None, + resource_reader: Callable[[], tuple[float, float, int | None]] | None = None, + ) -> None: ... + def finish(self) -> ProcessResourceUsage: ... + +class LintStopScope: + def __enter__(self) -> None: ... + def __exit__(self, *exception: object) -> bool: ... + +def lint_stop_scope(stop: Event, /) -> LintStopScope: ... +def active_lint_stop() -> Event | None: ... +def error_code(*, error: BaseException, fallback_code: str) -> str: ... +def error_help(error: BaseException, /) -> str | None: ... +def error_message(error: BaseException, /) -> str: ... +def setting_help( + *, purpose: str, file_name: str, section: str, key: str, value: bool | int | str +) -> str: ... +def snippet_help(*, purpose: str, target: str, lines: tuple[str, ...]) -> str: ... +def open_connection_with_hooks( + *, adapter: BaseAdapter, connection_config: dict[str, object], hooks: ConnectionHooks +) -> Any: ... +def build_analysis_cache_context( + *, + root: Path | None, + inference_profile: ExpressionInferenceProfile, + allow_compact_analysis: bool, + rich_type_inference: bool = True, + signature_namespace: object = None, +) -> AnalysisCacheContext | None: ... + +class CompileInputRecording: + def __enter__(self) -> CompileInputReads: ... + def __exit__(self, *exception: object) -> bool: ... + +def record_compile_input_reads() -> CompileInputRecording: ... +def record_settings_class_read(settings_class: type) -> None: ... +def report_compile_diagnostic(*, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None: ... +def compiler_cache_directory(project_dir: Path, /) -> Path: ... +def project_relative_path(*, path: Path, project_dir: Path) -> Path: ... def resolve_target_name( *, project_config: ProjectConfig, local_config: LocalConfig, selected_target: str | None ) -> str | None: ... @@ -3623,9 +3776,6 @@ def load_project_config_file(project_dir: Path, /) -> ProjectConfig: ... def load_local_config_file(project_dir: Path, /) -> LocalConfig: ... def resolve_config_file_path(project_dir: Path, local: bool, /) -> Path: ... def validate_local_sql_analysis(project_dir: Path, flags: tuple[bool, bool, bool], /) -> None: ... -def sql_test_cte_span( - contents: str, sql_body: str, cte_name: str, call: str | None, / -) -> tuple[int, int, int, int]: ... def normalize_scope_path(*, path: str | PurePath) -> str: ... def scope_index_fingerprint(*, project_dir: Path) -> str | None: ... def load_or_build_scope_index(*, project_dir: Path, no_cache: bool = False) -> ScopeIndex: ... @@ -3641,6 +3791,7 @@ class NativeConfigError: code: str | None help: str | None key: str | None + def exception(self, values: Mapping[str, object] | None = None) -> Exception: ... def parse_schema_columns( raw_columns: object, @@ -3915,6 +4066,17 @@ class SqlTestScanStore: def counts(self) -> tuple[int, int]: ... def save(self) -> tuple[int | None, str | None]: ... +class UnicodeEnvironment(Mapping[str, str]): + def __init__(self) -> None: ... + def __getitem__(self, name: str) -> str: ... + def __iter__(self) -> Iterator[str]: ... + def __len__(self) -> int: ... + +class SqlTestScanCache: + def __init__(self, *, cache_dir: Path | None) -> None: ... + def opened(self) -> SqlTestScanStore | None: ... + def save(self) -> None: ... + SQL_TEST_SCAN_STORE_FILE_NAME: str def content_digest(parts: list[str]) -> str: ... @@ -4528,9 +4690,6 @@ def sql_syntax_error( dialect: str | None, parse_one: bool = False, ) -> str | None: ... -def check_native_sql_syntax( - request: tuple[str, list[tuple[str, list[tuple[str, str]]]]], / -) -> bool: ... def assemble_native_project_resources( *, inputs: CompileInputsView, @@ -4538,10 +4697,10 @@ def assemble_native_project_resources( syntax_checks: tuple[tuple[tuple[str, dict[str, str] | None], ...], ...], ) -> NativeProjectResources: ... def explicit_reference_report( - project: Any, - settings: dict[str, object], - owners: list[tuple[Any, int, list[tuple[str, str, int]]]], - /, + *, + project: CompiledProjectView, + discovered_inputs: DiscoveredProjectInputs, + enforce_explicit: bool, ) -> PythonSqlReferenceReport: ... def hard_coded_relation_remedy( *, @@ -4647,9 +4806,6 @@ def schema_validations( *, requests: Sequence[SqlSchemaValidationRequest] ) -> tuple[SqlBindingResult, ...]: ... def ignores_quoted_case(*, connection: Mapping[str, object], dialect: str | None) -> bool: ... -def python_literal_sql_calls( - source: str, start_line: int, placeholder: str -) -> list[tuple[str, str, int]]: ... def _oracle_json_dumps(dialect_json: str, value_json: str) -> str: ... def _oracle_text_positions( data: bytes, byte_offsets: list[int] @@ -4797,6 +4953,66 @@ def evaluate_source_freshness_age_policy( data_version: object, observed_at: datetime, ) -> SourceFreshnessAgeStatus | None: ... +def require_aware_utc_datetime(*, value: datetime, field_name: str) -> datetime: ... +def build_source_freshness_read_latest_sql( + *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] +) -> str: ... +def build_source_freshness_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], +) -> str: ... +def read_latest_source_freshness( + *, + connection: object, + execute: AdapterExecute[Any, Any], + table_exists: bool, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_read_latest_sql: Callable[..., str], + source_names: tuple[str, ...] | None = None, +) -> SourceFreshnessSet: ... +def write_source_freshness_records( + *, + connection: object, + execute: AdapterExecute[Any, Any], + database: str | None, + schema: str, + records: tuple[SourceFreshnessRecord, ...], + renderers: SourceFreshnessRenderers, +) -> None: ... +def observe_configured_source_freshness( + *, adapter: StrictAdapter, connection: object, source: SourceEntry, observed_at: datetime +) -> SourceFreshnessObservation: ... +def observe_adapter_sources_freshness( + *, + adapter: StrictAdapter, + connection: object, + sources: tuple[SourceEntry, ...], + observed_at: datetime, +) -> AdapterSourceFreshnessBatch: ... +def source_freshness_record_from_observation( + *, observation: SourceFreshnessObservation, source: SourceEntry, run_id: str +) -> SourceFreshnessRecord: ... +def build_direct_source_freshness_planning_result( + *, + adapter: StrictAdapter, + connection: object, + sources: tuple[SourceEntry, ...], + state_database: str | None, + state_schemas: tuple[str, ...], + observed_at: datetime, + run_id: str, + render_qualified_name: Callable[..., str | None], + state_table_exists_by_schema: Mapping[str, bool], + filter_previous_to_sources: bool = False, +) -> DirectSourceFreshnessPlanningResult: ... +def build_direct_source_freshness_propagation_result( + *, source_freshness: DirectSourceFreshnessPlanningResult, scope: PlannerScope +) -> DirectSourceFreshnessPropagationResult: ... def redact_diagnostic_text(value: str) -> str: ... def redact_diagnostic_value(*, name: str, value: object) -> object: ... def sql_rule_lint_key(*, config: LintConfig) -> str: ... @@ -4819,7 +5035,7 @@ def cursor_sentinel_to_token(*, sentinel: Bound) -> str: ... def cursor_sentinel_from_token(*, token: str) -> BoundSentinel | None: ... def cursor_compare(*, left: CursorScalar, right: CursorScalar) -> int: ... def type_cursor_overrides( - *, cursor_overrides: CursorOverrides | None, selected_models: Iterable[CompiledModelView] + *, cursor_overrides: CursorOverrides | None, selected_models: Iterable[CompiledModelView] = () ) -> CursorOverrides | None: ... def maximum_allowed_start( *, discovered_value: str, cursor_grain: str | None, invocation_time: datetime, max_ahead: str @@ -4836,6 +5052,245 @@ def apply_maximum_start_policy( policy: MaximumStartPolicyInputs, has_start_override: bool, ) -> CursorBounds: ... +def build_plan_output( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + relations: PlannerRelationsContext, + changes: PlannerChangeResults, + model_entry_results: PlannerModelEntryResults, + reload_sources: bool, + extras: PlanOutputExtras | None = None, +) -> PlanOutput: ... +def plan_function( + *, + function: CompiledFunctionView, + adapter: BaseAdapter, + relations: PlannerRelationsContext, + fingerprint_query_sql: str, + previous_query_sql: str | None = None, + reason: PlanReason = ..., +) -> FunctionPlanEntry: ... +def plan_selected_sql_tests( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + selected_keys: frozenset[CompiledObjectKey], + sql_test_selection: SqlTestSelection, +) -> PlannedSqlTests: ... +def plan_selected_sql_tests_into( + *, + future: Future[PlannedSqlTests], + project: CompiledProjectView, + adapter: BaseAdapter, + selected_keys: frozenset[CompiledObjectKey], + sql_test_selection: SqlTestSelection, +) -> None: ... +def sql_test_is_selected( + *, + test: CompiledSqlTest, + selected_keys: frozenset[CompiledObjectKey], + sql_test_selection: SqlTestSelection, +) -> bool: ... +def build_selected_test_entries( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + selected_keys: frozenset[CompiledObjectKey], + sql_test_selection: SqlTestSelection, + case_name: str | None = None, +) -> tuple[list[SqlTestPlanEntry], list[PlanWarning]]: ... +def resolve_static_command_scope( + *, project: CompiledProjectView, selection: PlannerSelection, auto_load_sources: bool = False +) -> PlannerScope: ... +def resolve_static_relation_context( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + scope: PlannerScope, + deferral: DeferralInputs | None = None, + project_config: ProjectConfig | None = None, + local_config: LocalConfig | None = None, + relation_keys: frozenset[CompiledObjectKey] | None = None, +) -> PlannerRelationsContext: ... +def build_audit_command_plan( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + scope: PlannerScope, + relations: PlannerRelationsContext, +) -> PlanOutput: ... +def build_test_command_plan( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + scope: PlannerScope, + relations: PlannerRelationsContext, + case_name: str | None = None, +) -> PlanOutput: ... +def build_seed_command_plan( + *, + project: CompiledProjectView, + scope: PlannerScope, + relations: PlannerRelationsContext, + fingerprints: dict[str, Fingerprint] | None = None, +) -> PlanOutput: ... +def build_relation_command_plan( + *, project: CompiledProjectView, scope: PlannerScope, relations: PlannerRelationsContext +) -> PlanOutput: ... +def read_selected_seed_fingerprints( + *, adapter: BaseAdapter, connection: Any, seeds: tuple[CompiledSeedView, ...] +) -> dict[str, Fingerprint]: ... +def run_clone_planning( + *, + project: CompiledProjectView, + select: tuple[str, ...], + exclude: tuple[str, ...], + adapter: BaseAdapter, + connection: Any, + origin_project: CompiledProjectView, + origin_source_project: CompiledProjectView, + destination_source_project: CompiledProjectView, +) -> tuple[ + PlanOutput, + tuple[CloneSourcePlanEntry, ...], + tuple[ModelPlanEntry, ...], + tuple[SeedPlanEntry, ...], + tuple[FunctionPlanEntry, ...], + tuple[ModelPlanEntry, ...], + tuple[SeedPlanEntry, ...], + tuple[CloneSourcePlanEntry, ...], +]: ... +def resolve_clone_boundary[K]( + *, + selected: frozenset[K], + upstream: Mapping[K, tuple[K, ...]], + is_clonable: Callable[[K], bool], + is_view: Callable[[K], bool], +) -> frozenset[K]: ... +def resolve_skipped_view_chain[K]( + *, + selected: frozenset[K], + upstream: Mapping[K, tuple[K, ...]], + is_clonable: Callable[[K], bool], + is_view: Callable[[K], bool], +) -> frozenset[K]: ... +def sql_test_model_chain_names_by_key( + *, project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...] +) -> dict[CompiledObjectKey, tuple[str, ...]]: ... +def build_execution_plan( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + connection: Any, + selection: PlannerSelection, + overrides: PlannerOverrides, + deferral: DeferralInputs, + policies: PlannerPolicies, + on_progress: Callable[[str], None] | None = None, + project_config: ProjectConfig | None = None, + local_config: LocalConfig | None = None, +) -> PlanOutput: ... +def start_sql_test_planning( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + selected_keys: frozenset[CompiledObjectKey], + sql_test_selection: SqlTestSelection, + enabled: bool, +) -> Future[PlannedSqlTests]: ... +def build_display_only_sqlbuild_plan( + *, project: CompiledProjectView, selected_model_names: tuple[str, ...], full_refresh: bool +) -> PlanOutput: ... +def format_pre_build_work( + *, lines: list[str], plan: PlanOutput, display_options: DisplayOptions +) -> list[str]: ... +def resolve_future_cursor_config( + *, model: CompiledModelView, project_config: FutureCursorsConfig | None +) -> FutureCursorsConfig | None: ... +def resolve_start_cursor_config( + *, model: CompiledModelView, project_config: StartCursorsConfig | None +) -> StartCursorsConfig | None: ... +def future_cursor_cap_warning(bounds: CursorBounds | None, /) -> str | None: ... +def maximum_start_cap_warning(bounds: CursorBounds | None, /) -> str | None: ... +def compute_plan_cursor_bounds( + *, + model: CompiledModelView, + snapshot: WarehouseSnapshot, + backfill: BackfillResult, + full_refresh: bool, + start_cursor_override: str | None, + end_cursor_override: str | None, + runtime_owned_cursor_bounds: bool, + invocation_time: datetime | None = None, + future_cursor_config: FutureCursorsConfig | None = None, + start_cursor_config: StartCursorsConfig | None = None, + replaces_relation: bool = False, + cursor_input_relations: tuple[CursorInputRelation, ...] = (), +) -> CursorBounds | None: ... +def compute_microbatch_range( + *, + model: CompiledModelView, + snapshot: WarehouseSnapshot, + backfill: BackfillResult, + full_refresh: bool, + start_cursor_override: str | None, + end_cursor_override: str | None, + runtime_owned_cursor_bounds: bool, + invocation_time: datetime | None = None, + future_cursor_config: FutureCursorsConfig | None = None, + start_cursor_config: StartCursorsConfig | None = None, + replaces_relation: bool = False, + cursor_input_relations: tuple[CursorInputRelation, ...] = (), +) -> CursorBounds | None: ... +def resolve_empty_cursor_inputs( + *, + model: CompiledModelView, + snapshot: WarehouseSnapshot, + full_refresh: bool, + runtime_owned_cursor_bounds: bool, + start_cursor_override: str | None, + end_cursor_override: str | None, +) -> tuple[tuple[str, ...], bool]: ... +def resolve_cursor_overrides( + *, + model: CompiledModelView, + cursor_overrides: CursorOverrides | None, + start_cursor_override: str | None, + end_cursor_override: str | None, +) -> tuple[str | None, str | None]: ... +def build_planner_relations_context( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + connection: Any, + scope: PlannerScope, + deferral: DeferralInputs | None = None, + project_config: ProjectConfig | None = None, + local_config: LocalConfig | None = None, + known_source_columns: dict[str, tuple[ColumnInfo, ...]] | None = None, + relation_keys: frozenset[CompiledObjectKey] | None = None, +) -> PlannerRelationsContext: ... +def build_plan_entries( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + relations: PlannerRelationsContext, + resolved_actions: PlannerResolvedActions, + cursor_overrides: CursorOverrides | None, + full_refresh: bool, + build_inputs: PlanEntryBuildInputs | None = None, +) -> PlannerModelEntryResults: ... +def build_model_materializations( + model_entries: tuple[ModelPlanEntry, ...], / +) -> dict[str, str]: ... +def scope_overlaps( + *, scope_deps: tuple[CompiledObjectKey, ...], selected_keys: frozenset[CompiledObjectKey] +) -> bool: ... def resolve_future_cursor_safety( *, bounds: CursorBounds, @@ -5790,3 +6245,96 @@ def list_source_candidates( database: str | None, entries: tuple[SourceEntry, ...], ) -> tuple[RelationInfo, ...]: ... + +class CostArtifactError(ValueError): ... +class SourceFreshnessInputError(ValueError): ... +class SourceFreshnessObservationError(ValueError): ... + +class CostTelemetryHealth: + def mark_ledger_failure(self, *, run_id: str, error: BaseException) -> None: ... + def consume_ledger_failure(self, *, run_id: str) -> str | None: ... + +COST_TELEMETRY_HEALTH: CostTelemetryHealth + +class CostScope: + def __enter__(self) -> None: ... + def __exit__(self, *exception: object) -> None: ... + +class CostContext: + @staticmethod + def current() -> CostResourceContext | None: ... + @staticmethod + def scope( + *, + run_id: str, + resource_type: str, + resource_name: str, + ledger_path: Path | None = None, + phase: str = "execute", + attempt: int = 1, + on_statement_complete: Callable[[StatementExecutionTelemetry], None] | None = None, + ) -> CostScope: ... + @staticmethod + def resource_scope( + *, + resource_type: str, + resource_name: str, + phase: str | None = None, + attempt: int = 1, + ) -> CostScope: ... + +def allocate_run_cost( + *, + observations: Iterable[QueryCostObservation], + run_id: str, + usd_per_credit: Decimal, + rate_source: str, + result_limit_reached: bool = False, +) -> RunCostSummary: ... +def record_statement( + *, + context: CostResourceContext, + statement_id: str, + sql: str, + query_id: object, + status: str, + started_at: datetime, + completed_at: datetime, + error: BaseException | None = None, +) -> None: ... +def read_statement_ledger(*, path: Path, run_id: str) -> tuple[StatementLedgerEntry, ...]: ... +def collect_snowflake_cost( + *, + connection: Any, + run_id: str, + started_at: datetime, + completed_at: datetime, + statement_ledger_path: Path, + usd_per_credit: Decimal, + rate_source: str, + database: str | None = None, + attempts: int = 3, + retry_delay_seconds: float = 1.0, + result_limit: int = 10000, +) -> RunCostSummary: ... +def snowflake_query_observations( + *, + connection: Any, + started_at: datetime, + completed_at: datetime, + expected_query_ids: Iterable[str], + classification_query_ids: Iterable[str], + deadline: float, + database: str | None = None, + result_limit: int | None = None, +) -> tuple[tuple[QueryCostObservation, ...], frozenset[str], dict[str, str], bool]: ... +def render_snowflake_query_history_sql( + *, + started_at: datetime, + classification_query_ids: Iterable[str], + database: str | None = None, + completed_at: datetime | None = None, + result_limit: int = 10000, +) -> str: ... +def snowflake_statement_is_metadata_only(*, query_type: object, query_text: object) -> bool: ... +def parse_sqlbuild_query_tag(value: object) -> dict[str, str | None] | None: ... diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 9578d2fc0b..cb33a480c7 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -2166,11 +2166,9 @@ def render_read_latest_source_freshness_sql( ) -> str: """Render SQL that reads latest source freshness rows per identity.""" - from sqlbuild.compiler.source_freshness.main.read_latest_sql import ( - build_read_latest_sql, - ) + from sqlbuild._native import build_source_freshness_read_latest_sql - return build_read_latest_sql( + return build_source_freshness_read_latest_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -2190,9 +2188,7 @@ def render_create_fingerprint_index_sqls( def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: """Render DDL that creates the source freshness table when it is missing.""" - from sqlbuild.compiler.source_freshness.main.create_table_sql import ( - build_create_table_sql as build_source_freshness_create_table_sql, - ) + from sqlbuild._native import build_source_freshness_create_table_sql return build_source_freshness_create_table_sql( database=database, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index cf6a389d47..ec8ecff6f5 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -807,20 +807,16 @@ def render_read_latest_source_freshness_sql( database: str | None, schema: str, ) -> str: - from sqlbuild.compiler.source_freshness.main.read_latest_sql import ( - build_read_latest_sql, - ) + from sqlbuild._native import build_source_freshness_read_latest_sql - return build_read_latest_sql( + return build_source_freshness_read_latest_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, ) def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.source_freshness.main.create_table_sql import ( - build_create_table_sql as build_source_freshness_create_table_sql, - ) + from sqlbuild._native import build_source_freshness_create_table_sql return build_source_freshness_create_table_sql( database=database, diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 8184bbba30..4601cc88e8 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -261,20 +261,16 @@ def render_read_latest_source_freshness_sql( database: str | None, schema: str, ) -> str: - from sqlbuild.compiler.source_freshness.main.read_latest_sql import ( - build_read_latest_sql, - ) + from sqlbuild._native import build_source_freshness_read_latest_sql - return build_read_latest_sql( + return build_source_freshness_read_latest_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, ) def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.source_freshness.main.create_table_sql import ( - build_create_table_sql as build_source_freshness_create_table_sql, - ) + from sqlbuild._native import build_source_freshness_create_table_sql return build_source_freshness_create_table_sql( database=database, diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 478eaf5c33..a15d065224 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -357,20 +357,16 @@ def render_read_latest_source_freshness_sql( database: str | None, schema: str, ) -> str: - from sqlbuild.compiler.source_freshness.main.read_latest_sql import ( - build_read_latest_sql, - ) + from sqlbuild._native import build_source_freshness_read_latest_sql - return build_read_latest_sql( + return build_source_freshness_read_latest_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, ) def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.source_freshness.main.create_table_sql import ( - build_create_table_sql as build_source_freshness_create_table_sql, - ) + from sqlbuild._native import build_source_freshness_create_table_sql return build_source_freshness_create_table_sql( database=database, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 8eb92916f0..5cd919f515 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -1593,9 +1593,7 @@ def render_prune_fingerprint_history_sql( ) def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.source_freshness.main.create_table_sql import ( - build_create_table_sql as build_source_freshness_create_table_sql, - ) + from sqlbuild._native import build_source_freshness_create_table_sql return build_source_freshness_create_table_sql( database=database, @@ -1770,11 +1768,9 @@ def render_read_latest_source_freshness_sql( database: str | None, schema: str, ) -> str: - from sqlbuild.compiler.source_freshness.main.read_latest_sql import ( - build_read_latest_sql, - ) + from sqlbuild._native import build_source_freshness_read_latest_sql - return build_read_latest_sql( + return build_source_freshness_read_latest_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 826a015573..c20afa3e80 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -12,6 +12,7 @@ from pathlib import Path from typing import Any, ClassVar, cast +from sqlbuild._native import collect_snowflake_cost from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _encode_typed_json, @@ -152,7 +153,6 @@ from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.cost.main.collection import collect_snowflake_cost from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostCapability, CostStatus from sqlbuild.diagnostics.main.log_sql import log_sql @@ -533,20 +533,16 @@ def render_read_latest_source_freshness_sql( database: str | None, schema: str, ) -> str: - from sqlbuild.compiler.source_freshness.main.read_latest_sql import ( - build_read_latest_sql, - ) + from sqlbuild._native import build_source_freshness_read_latest_sql - return build_read_latest_sql( + return build_source_freshness_read_latest_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, ) def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.source_freshness.main.create_table_sql import ( - build_create_table_sql as build_source_freshness_create_table_sql, - ) + from sqlbuild._native import build_source_freshness_create_table_sql return self._with_state_table_retention( create_sql=build_source_freshness_create_table_sql( diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_cursor.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_cursor.py index 712fe1b1f6..8b5d0fd682 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_cursor.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_cursor.py @@ -12,8 +12,7 @@ from typing import Any from uuid import uuid4 -from sqlbuild.cost.classes.cost_context import CostContext -from sqlbuild.cost.classes.statement_ledger import StatementLedger +from sqlbuild._native import CostContext, record_statement from sqlbuild.cost.constants import SQLBUILD_QUERY_TAG_APP from sqlbuild.cost.models import CostResourceContext, StatementExecutionTelemetry from sqlbuild.runtime.execution_limits.main.current_execution_deadline import ( @@ -224,7 +223,7 @@ def _record_failure( cursor=self.raw_cursor, previous_query_id=previous_query_id ) if context is not None: - StatementLedger.record( + record_statement( context=context, statement_id=statement_id, sql=sql, @@ -258,7 +257,7 @@ def _record_success( cursor=self.raw_cursor, previous_query_id=previous_query_id ) if context is not None: - StatementLedger.record( + record_statement( context=context, statement_id=statement_id, sql=sql, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 2dcddfb1e7..ccfda8d0b2 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -448,9 +448,7 @@ def render_prune_fingerprint_history_sql( ) def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.source_freshness.main.create_table_sql import ( - build_create_table_sql as build_source_freshness_create_table_sql, - ) + from sqlbuild._native import build_source_freshness_create_table_sql return build_source_freshness_create_table_sql( database=database, @@ -720,11 +718,9 @@ def render_read_latest_source_freshness_sql( database: str | None, schema: str, ) -> str: - from sqlbuild.compiler.source_freshness.main.read_latest_sql import ( - build_read_latest_sql, - ) + from sqlbuild._native import build_source_freshness_read_latest_sql - return build_read_latest_sql( + return build_source_freshness_read_latest_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py index 90fd1430ad..3741f1a35f 100644 --- a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py +++ b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py @@ -5,16 +5,15 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import CostContext, write_source_freshness_records from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput -from sqlbuild.compiler.source_freshness.main.write import write_source_freshness_records from sqlbuild.compiler.source_freshness.models import ( DirectSourceFreshnessPlanningResult, SourceFreshnessIdentity, SourceFreshnessRecord, SourceFreshnessRenderers, ) -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/cli/commands/constants.py b/src/sqlbuild/cli/commands/constants.py deleted file mode 100644 index a0d40b595d..0000000000 --- a/src/sqlbuild/cli/commands/constants.py +++ /dev/null @@ -1,112 +0,0 @@ -"""CLI command decision constants.""" - -from types import MappingProxyType - -from sqlbuild.cli.commands.types import CliCommand -from sqlbuild.spec.contracts.types import WarehouseGroup - -C0_CONTROL_CODE_LIMIT: int = 32 -C1_CONTROL_CODE_START: int = 127 -C1_CONTROL_CODE_LIMIT: int = 160 -EMPTY_DAG_PATH: str = "" -TARGET_DIRECTORY_NAME: str = "target" -SQLBUILD_CONCURRENCY_ENV_VAR: str = "SQLBUILD_CONCURRENCY" -NO_COLOR_OPTION: str = "--no-color" -DBT_INIT_COMMAND: str = "init" -SCENARIO_TEST_COMMAND: str = "test" -SCENARIO_CAPTURE_COMMAND: str = "capture" -JSON_OUTPUT_FORMAT: str = "json" -CSV_OUTPUT_FORMAT: str = "csv" -DEBUG_COMMAND: str = "debug" -LINEAGE_COMMAND: str = "lineage" -QUERY_COMMAND: str = "query" -RULES_LIST_ACTION: str = "list" -RULES_SHOW_ACTION: str = "show" -RULES_SKILLS_ACTION: str = "skills" -SQL_FILE_SUFFIX: str = ".sql" -SUCCESS_STATUS: str = "success" -FAILED_STATUS: str = "failed" -SCENARIO_CLI_MISSING_SUBCOMMAND: str = "C450" -SQL_ANALYSIS_CONFIG_KEY: str = "sql_analysis" -COST_NARROW_TERMINAL_WIDTH: int = 120 -COST_HISTORY_NUMERIC_COLUMNS: frozenset[int] = frozenset({4, 5}) -COST_RESOURCE_NUMERIC_START_COLUMN: int = 2 -BYTE_SCALE: int = 1024 -BYTE_UNIT: str = "B" -TERABYTE_UNIT: str = "TiB" -SCOPE_PATH_RELATIVE: str = "relative" -SCOPE_PATH_COMPACT: str = "compact" -SCOPE_PATH_NONE: str = "none" -SCOPE_GLOBAL_SUMMARY: str = "summary" -SCOPE_GLOBAL_ALL: str = "all" -SCOPE_DEFAULT_PAGE_SIZE: int = 100 -MISSING_ORIGIN_BUILD_HELP: str = ( - "Pass --allow-missing-migration-origin to confirm in non-interactive runs." -) -MISSING_ORIGIN_DBT_HELP: str = ( - "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected " - "models with sqb build --allow-missing-migration-origin and leave them out of this dbt " - "selection, or set the target's missing_migration_origin to 'allow'." -) -SNAPSHOT_FULL_REFRESH_BUILD_HELP: str = ( - "Pass --allow-snapshot-full-refresh to confirm in non-interactive runs." -) -SNAPSHOT_FULL_REFRESH_DBT_HELP: str = ( - "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected " - "snapshots with sqb build --allow-snapshot-full-refresh and leave them out of this dbt " - "selection, or relax the snapshot_full_refresh policy." -) -TABLE_TYPE_DOWNGRADE_BUILD_HELP: str = ( - "Pass --allow-table-type-downgrade to confirm in non-interactive runs." -) -TABLE_TYPE_DOWNGRADE_DBT_HELP: str = ( - "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected " - "models with sqb build --allow-table-type-downgrade and leave them out of this dbt " - "selection, or set the target's table_type_downgrade to 'allow'." -) -RETENTION_DECREASE_BUILD_HELP: str = ( - "Pass --allow-retention-decrease to confirm in non-interactive runs." -) -RETENTION_DECREASE_DBT_HELP: str = ( - "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected " - "models with sqb build --allow-retention-decrease and leave them out of this dbt " - "selection, or set the target's time_travel_retention_decrease to 'allow'." -) -EXECUTION_LIMIT_BUILD_NOTE: str = "No warehouse changes were made." -EXECUTION_LIMIT_DBT_NOTE: str = "dbt work has already run; no SQLBuild models were built." -CLI_WAREHOUSE_SOURCE: str = "--warehouse" -CONNECTION_WAREHOUSE_SOURCE: str = "connection warehouse" -CONNECTION_DEFAULT_WAREHOUSE_SOURCE: str = "connection default" -COMMAND_WAREHOUSE_GROUPS: MappingProxyType[CliCommand, WarehouseGroup | None] = MappingProxyType( - { - CliCommand.AUDIT: WarehouseGroup.BUILD, - CliCommand.BUILD: WarehouseGroup.BUILD, - CliCommand.CHECK: WarehouseGroup.BUILD, - CliCommand.CLONE: WarehouseGroup.BUILD, - CliCommand.DBT: WarehouseGroup.BUILD, - CliCommand.LOAD: WarehouseGroup.BUILD, - CliCommand.PLAN: WarehouseGroup.BUILD, - CliCommand.SEED: WarehouseGroup.BUILD, - CliCommand.CONTRACT: WarehouseGroup.QUERY, - CliCommand.COST: WarehouseGroup.QUERY, - CliCommand.DEBUG: WarehouseGroup.QUERY, - CliCommand.DIFF: WarehouseGroup.QUERY, - CliCommand.FRESHNESS: WarehouseGroup.QUERY, - CliCommand.QUERY: WarehouseGroup.QUERY, - CliCommand.JANITOR: WarehouseGroup.QUERY, - CliCommand.SCENARIO: WarehouseGroup.QUERY, - CliCommand.TEST: WarehouseGroup.QUERY, - CliCommand.CLEAN: None, - CliCommand.COMPILE: None, - CliCommand.DAG: None, - CliCommand.FORMAT: None, - CliCommand.INIT: None, - CliCommand.LINEAGE: None, - CliCommand.MV: None, - CliCommand.PLAYGROUND: None, - CliCommand.RENAME: None, - CliCommand.RULES: None, - CliCommand.SCOPE: None, - CliCommand.SKILLS: None, - } -) diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index a6c9d8cca0..c115ceb35f 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -1,80 +1,6 @@ -"""Compile reuse storage, fingerprint, and environment constants.""" +"""Compile reuse runtime identity constants.""" from __future__ import annotations -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR, - STAGE_CAPTURE_DIR_ENV_VAR, -) - REUSE_FORMAT_VERSION: int = 1 -RETIRED_REUSE_DIRECTORY_NAME: str = "project-reuse-v1" -NATIVE_REUSE_DIRECTORY_NAME: str = "project-reuse-v2" -NATIVE_REUSE_SUFFIX: str = ".store" -REUSE_MAX_ENTRY_BYTES: int = 512 * 1024 * 1024 -REUSE_MAX_STORED_ENTRIES: int = 8 -REUSE_DISABLE_ENV_VAR: str = COMPILE_REUSE_DISABLE_ENV_VAR -REUSE_DISABLE_VALUE: str = "1" -REUSE_LOGGER_NAME: str = "sqlbuild.compile.reuse" -TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] = ("SQLBUILD_", "SQB_") -UNTRACKED_ENVIRONMENT_NAMES: frozenset[str] = frozenset( - { - COMPILE_REUSE_DISABLE_ENV_VAR, - STAGE_CAPTURE_DIR_ENV_VAR, - } -) -MISSING_ENVIRONMENT_VALUE: str = "\0missing" -MISSING_FILE_DIGEST: str = "\0missing" -PROJECT_CONFIG_FILENAMES: tuple[str, ...] = ("sqlbuild_project.toml", "sqlbuild_project.yml") - -RACY_WINDOW_NS: int = 2_000_000_000 -MISSING_PATH_MTIME_NS: int = -1 -PROJECT_ROOT_PATH_MTIME_NS: int = -2 OUTPUT_FILE_DESCRIPTORS: tuple[int, ...] = (1, 2) - -TARGET_DIRECTORY_NAME: str = "target" -COMPILED_DIRECTORY_NAME: str = "compiled" -EXCLUDED_ROOT_DIRECTORIES: frozenset[str] = frozenset( - { - "logs", - "target", - "venv", - ".cache", - ".fensu", - ".hg", - ".idea", - ".mypy_cache", - ".nox", - ".pytest_cache", - ".ruff_cache", - ".sqlbuild", - ".svn", - ".tox", - ".venv", - ".vscode", - } -) -EXCLUDED_DIRECTORIES: frozenset[str] = frozenset({".git", "__pycache__"}) -PRESENCE_ONLY_FILE_SUFFIXES: tuple[str, ...] = (".duckdb", ".duckdb.wal") - -REUSE_HIT_TIMING: str = "project_reuse_hits" -REUSE_MISS_TIMING: str = "project_reuse_misses" -REUSE_BYPASS_TIMING: str = "project_reuse_bypasses" -REUSE_CHECK_TIMING: str = "project_reuse_check_ms" -TOTAL_TIMING: str = "total_ms" -REUSE_HIT_MESSAGE: str = "Inputs unchanged; reused the previous compile ({seconds:.1f} s)" -MILLISECONDS_PER_SECOND: int = 1000 -REUSE_STORE_NOTICE_BYTES: int = 64 * 1024 * 1024 -REUSE_STORE_NOTICE_PATHS: int = 100_000 -BYTES_PER_MEBIBYTE: int = 1024 * 1024 -REUSE_STORE_START_MESSAGE: str = ( - "Recording compile for reuse ({paths} paths, {mebibytes:.0f} MiB to hash)..." -) -REUSE_STORE_DONE_MESSAGE: str = "Recorded compile for reuse ({seconds:.1f} s)" -REUSE_STORE_SKIPPED_MESSAGE: str = ( - "Did not record compile for reuse: files changed while it ran ({seconds:.1f} s)" -) -LAUNCH_RECORD_FORMAT: int = 1 -LAUNCH_RECORD_DIRECTORY: tuple[str, ...] = (TARGET_DIRECTORY_NAME, "cache", "launcher-v1") -COMPUTE_LOG_DIRECTORY_NAME: str = "logs" -COMPILE_COMMAND_NAME: str = "compile" diff --git a/src/sqlbuild/cli/compile_reuse/models.py b/src/sqlbuild/cli/compile_reuse/models.py index 5ea1860000..6c1af6b816 100644 --- a/src/sqlbuild/cli/compile_reuse/models.py +++ b/src/sqlbuild/cli/compile_reuse/models.py @@ -1,13 +1,10 @@ -"""Compile reuse requests, provider settings inputs, and per-invocation reuse state.""" +"""Compile reuse requests and provider settings inputs.""" from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass from pathlib import Path -from sqlbuild._native import NativeReuseAttempt -from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome - @dataclass(frozen=True) class CompileReuseRequest: @@ -47,19 +44,3 @@ class SettingsInputsResult: inputs: tuple[SettingsEnvironmentInputs, ...] unsupported_reason: str | None = None - - -@dataclass(frozen=True) -class CompileReuseAttempt: - """Result of checking for a reusable compile, carried into a full compile on a miss.""" - - outcome: CompileReuseOutcome - project_dir: Path - started: float - check_ms: int = 0 - exit_code: int | None = None - entry_path: Path | None = None - native: NativeReuseAttempt | None = field(default=None, compare=False, repr=False) - """The natively checked attempt, recorded natively after a miss.""" - selected_target: str | None = None - json_output: bool = False diff --git a/src/sqlbuild/cli/compile_reuse/types.py b/src/sqlbuild/cli/compile_reuse/types.py deleted file mode 100644 index c592c4e642..0000000000 --- a/src/sqlbuild/cli/compile_reuse/types.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Compile reuse outcome types.""" - -from __future__ import annotations - -from enum import StrEnum - - -class CompileReuseOutcome(StrEnum): - """How one compile invocation used the stored previous compile.""" - - HIT = "hit" - MISS = "miss" - BYPASS = "bypass" diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/cache.py b/src/sqlbuild/compiler/compile/_helpers/analysis/cache.py deleted file mode 100644 index 2ae479e965..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/cache.py +++ /dev/null @@ -1,122 +0,0 @@ -"""The analysis cache context and the model analysis cache metrics.""" - -from __future__ import annotations - -import hashlib -import inspect -import json -import platform -from importlib.metadata import PackageNotFoundError, version -from pathlib import Path - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import AnalysisCacheContext -from sqlbuild.compiler.profiling.main._metric import record_compile_metric -from sqlbuild.compiler.sql_analysis.constants import TYPE_CHECKED_DIALECTS - -_ANALYSIS_CACHE_VERSION: int = 13 -_ANALYSIS_ALGORITHM_FINGERPRINT: str = "model-sql-analysis-v21-completed-star-shapes" -_LOCAL_QUALNAME_MARKER: str = "" - - -def build_analysis_cache_context( - *, - root: Path | None, - inference_profile: ExpressionInferenceProfile, - allow_compact_analysis: bool, - rich_type_inference: bool = True, - signature_namespace: object = None, -) -> AnalysisCacheContext | None: - """Build a reusable cache context, or bypass when profile identity is unstable.""" - - if root is None: - return None - profile_payload: dict[str, object] | None = _inference_profile_payload(inference_profile) - if profile_payload is None: - return None - shared_payload: dict[str, object] = { - "algorithm": _ANALYSIS_ALGORITHM_FINGERPRINT, - "cache_version": _ANALYSIS_CACHE_VERSION, - "sqlbuild_version": _package_version("sqlbuild"), - "polyglot_version": _package_version("polyglot-sql-chio"), - "python_version": platform.python_version_tuple()[:2], - "allow_compact_analysis": allow_compact_analysis, - "rich_type_inference": rich_type_inference, - "type_checked_dialects": sorted(TYPE_CHECKED_DIALECTS), - "inference_profile": profile_payload, - } - try: - return AnalysisCacheContext( - root=root, - shared_fingerprint=_payload_digest(shared_payload), - signature_namespace=_payload_digest(signature_namespace), - ) - except (TypeError, ValueError): - return None - - -def record_analysis_cache_metrics( - *, batch_hits: int, entry_hits: int, misses: int, bypasses: int -) -> None: - """Record semantic analysis cache outcomes for one invocation.""" - - record_compile_metric(metric="analysis_batch_cache_hits", value=batch_hits) - record_compile_metric(metric="analysis_entry_cache_hits", value=entry_hits) - record_compile_metric(metric="analysis_cache_misses", value=misses) - record_compile_metric(metric="analysis_cache_bypasses", value=bypasses) - - -def _inference_profile_payload( - profile: ExpressionInferenceProfile, -) -> dict[str, object] | None: - rules: list[dict[str, str]] = [] - for name, rule in sorted(profile.function_nullability_rules.items()): - if not inspect.isfunction(rule): - return None - module: str = rule.__module__ - qualname: str = rule.__qualname__ - if ( - not module.startswith("sqlbuild.") - or _LOCAL_QUALNAME_MARKER in qualname - or rule.__closure__ is not None - or rule.__defaults__ is not None - or rule.__kwdefaults__ is not None - ): - return None - try: - source: str = inspect.getsource(rule) - except (OSError, TypeError): - return None - rules.append( - { - "name": name, - "module": module, - "qualname": qualname, - "source_digest": hashlib.sha256(source.encode("utf-8")).hexdigest(), - } - ) - return { - "sql_analysis_dialect": profile.sql_analysis_dialect, - "quoted_identifiers_ignore_case": profile.quoted_identifiers_ignore_case, - "semantic_known_functions": profile.semantic_known_functions, - "semantic_known_types": profile.semantic_known_types, - "function_nullability_rules": rules, - "function_return_types": dict(sorted(profile.function_return_types.items())), - } - - -def _payload_digest(payload: object) -> str: - encoded: bytes = json.dumps( - payload, - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") - return hashlib.sha256(encoded).hexdigest() - - -def _package_version(package: str) -> str: - try: - return version(package) - except PackageNotFoundError: - return "unknown" diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py deleted file mode 100644 index 9ec47b3b56..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Collect compile diagnostics so one compile reports every violation, not only the first.""" - -from __future__ import annotations - -from collections.abc import Iterator -from contextlib import contextmanager -from contextvars import ContextVar, Token - -from sqlbuild.compiler.compile.classes.collected_compile_diagnostics import ( - CollectedCompileDiagnostics, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompilerDiagnostic, -) - -_ACTIVE_COLLECTOR: ContextVar[CollectedCompileDiagnostics | None] = ContextVar( - "sqlbuild_collected_compile_diagnostics", default=None -) - - -@contextmanager -def collect_compile_diagnostics() -> Iterator[CollectedCompileDiagnostics]: - """Collect diagnostics reported inside the block instead of raising the first one.""" - - collector: CollectedCompileDiagnostics = CollectedCompileDiagnostics() - token: Token[CollectedCompileDiagnostics | None] = _ACTIVE_COLLECTOR.set(collector) - try: - yield collector - finally: - _ACTIVE_COLLECTOR.reset(token) - - -def report_compile_diagnostic(*, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None: - """Record a diagnostic in the active collector, or raise it when none is active.""" - - collector: CollectedCompileDiagnostics | None = _ACTIVE_COLLECTOR.get() - if collector is not None: - collector.add(key=key, diagnostic=diagnostic) - return - location: str = ( - f"\n --> {diagnostic.location.path.as_posix()}:{diagnostic.location.line}" - if diagnostic.location is not None - else f"\n --> {diagnostic.path.as_posix()}" - if diagnostic.path is not None - else "" - ) - raise CompileInputError( - f"{diagnostic.message}{location}", code=diagnostic.code, help=diagnostic.help - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py index 18f136d0d9..778eaaefb4 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py @@ -7,9 +7,9 @@ from functools import partial from pathlib import Path +from sqlbuild._native import snippet_help from sqlbuild.compiler.compile.types import PythonResourceRow, SourceInput from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredProvider -from sqlbuild.errors.setting_help.main.snippet_help import snippet_help from sqlbuild.spec.contracts.models import SourceLocation diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py deleted file mode 100644 index 6859c703c8..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Compile's explicit-reference checks: Python collects user-code owners, native decides.""" - -from __future__ import annotations - -import inspect -import textwrap -from collections.abc import Callable, Mapping -from dataclasses import dataclass, field -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild._native import explicit_references_help, model_python_hook_names -from sqlbuild.compiler.compile.models import PythonSqlReferenceReport -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind -from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition -from sqlbuild.python_nodes.models import LoaderDefinition, SqlResourceRef -from sqlbuild.python_nodes.types import SqlResourceRefKind - -_VALUE_PLACEHOLDER: str = "__sqlbuild_python_value_" - - -@dataclass(frozen=True) -class _PythonSqlOwner: - label: str - kind: HardCodedRelationOwnerKind - function: Callable[..., object] - relative_path: Path - own_refs: frozenset[SqlResourceRef] = frozenset() - upstream_loader_by_source: Mapping[str, str] = field(default_factory=dict) - - -def explicit_reference_report( - *, - project: CompiledProjectView, - discovered_inputs: DiscoveredProjectInputs, - enforce_explicit: bool, -) -> PythonSqlReferenceReport: - """Return P005, P007 and P008 errors and unmatched Python-SQL relations; raise P001.""" - - owners: tuple[_PythonSqlOwner, ...] = ( - _owners(project=project, discovered_inputs=discovered_inputs) if enforce_explicit else () - ) - paths: list[Path] = sorted({owner.relative_path for owner in owners}) - path_order: dict[Path, int] = {path: index for index, path in enumerate(paths)} - return _native.explicit_reference_report( - project, - { - "retains_old_names": discovered_inputs.project_config.migrations.old_name_views - is not None, - "enforce_explicit": enforce_explicit, - "explicit_help": ( - explicit_references_help(allowed="hard-coded relation names") - if enforce_explicit - else "" - ), - "placeholder": _VALUE_PLACEHOLDER, - }, - [ - (owner, path_order[owner.relative_path], _literal_sql_calls(owner.function)) - for owner in owners - ], - ) - - -def _owners( - *, project: CompiledProjectView, discovered_inputs: DiscoveredProjectInputs -) -> tuple[_PythonSqlOwner, ...]: - owners: list[_PythonSqlOwner] = [] - for label, nodes in ( - ("task", discovered_inputs.task_functions), - ("asset", discovered_inputs.asset_functions), - ("check", discovered_inputs.check_functions), - ): - owners.extend( - _PythonSqlOwner( - label=f"{label}:{node.name}", - kind=HardCodedRelationOwnerKind.NODE, - function=node.function, - relative_path=node.relative_path, - ) - for node in nodes - ) - owners.extend(_loader_owners(project=project, discovered_inputs=discovered_inputs)) - attached_models: dict[str, set[SqlResourceRef]] = {} - for model in project.models: - for hook_name in model_python_hook_names(model=model): - attached_models.setdefault(hook_name, set()).add( - SqlResourceRef(kind=SqlResourceRefKind.MODEL, name=model.name) - ) - owners.extend( - _PythonSqlOwner( - label=f"hook:{hook.name}", - kind=HardCodedRelationOwnerKind.HOOK, - function=hook.function, - relative_path=hook.relative_path, - own_refs=frozenset(attached_models.get(hook.name, ())), - ) - for hook in project.hook_functions - ) - return tuple(owners) - - -def _loader_owners( - *, project: CompiledProjectView, discovered_inputs: DiscoveredProjectInputs -) -> tuple[_PythonSqlOwner, ...]: - sources_by_loader: dict[str, set[str]] = {} - for source in project.sources: - if source.source_entry.loader is not None: - sources_by_loader.setdefault(source.source_entry.loader, set()).add(source.name) - owners: list[_PythonSqlOwner] = [] - for loader in discovered_inputs.loader_functions: - upstream_loader_by_source: dict[str, str] = {} - for dependency in loader.depends_on: - definition: LoaderDefinition | None = ( - read_loader_definition(dependency) if callable(dependency) else None - ) - if definition is None: - continue - for source_name in sources_by_loader.get(definition.name, ()): - upstream_loader_by_source[source_name] = getattr( - dependency, "__name__", definition.name - ) - owners.append( - _PythonSqlOwner( - label=f"loader:{loader.name}", - kind=HardCodedRelationOwnerKind.LOADER, - function=loader.function, - relative_path=loader.relative_path, - own_refs=frozenset( - SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=source_name) - for source_name in sources_by_loader.get(loader.name, ()) - ), - upstream_loader_by_source=upstream_loader_by_source, - ) - ) - return tuple(owners) - - -def _literal_sql_calls(function: Callable[..., object]) -> list[tuple[str, str, int]]: - try: - lines: list[str] - start_line: int - lines, start_line = inspect.getsourcelines(function) - except (OSError, TypeError): - return [] - return _native.python_literal_sql_calls( - textwrap.dedent("".join(lines)), start_line, _VALUE_PLACEHOLDER - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py deleted file mode 100644 index d2f2fac5da..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Typed reference arguments and explicit-reference enforcement for macro expansion.""" - -from __future__ import annotations - -from sqlbuild._native import extract_sql_references -from sqlbuild.compiler.compile.models import ( - CompileSqlReference, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.python_nodes.types import SqlResourceRefKind - -_TYPED_REFERENCE_KINDS: dict[str, SqlResourceRefKind] = { - "__ref": SqlResourceRefKind.MODEL, - "__source": SqlResourceRefKind.SOURCE, - "__seed": SqlResourceRefKind.SEED, -} -_REFERENCE_FUNCTION_BY_KIND: dict[SqlResourceRefKind, str] = { - kind: function for function, kind in _TYPED_REFERENCE_KINDS.items() -} - -_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() - - -def reference_call_text(ref: SqlResourceRef) -> str: - """Return the SQL reference call that names ``ref``, such as ``__ref("orders")``.""" - - return f'{_REFERENCE_FUNCTION_BY_KIND[ref.kind]}("{ref.name}")' - - -def call_site_sql_references( - refs: tuple[SqlResourceRef, ...], -) -> tuple[CompileSqlReference, ...]: - """Return the SQL references written as typed macro arguments, as SQL extraction sees them.""" - - if not refs: - return () - return extract_sql_references( - sql=" ".join(reference_call_text(ref) for ref in refs), syntax=_GENERIC_SQL_SYNTAX - ) - - -def typed_reference_kinds() -> dict[str, SqlResourceRefKind]: - """Each typed reference function, such as `__ref`, and the resource kind it names.""" - - return dict(_TYPED_REFERENCE_KINDS) diff --git a/src/sqlbuild/compiler/compile/classes/__init__.py b/src/sqlbuild/compiler/compile/classes/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py b/src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py deleted file mode 100644 index 9a1a32e725..0000000000 --- a/src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Compile diagnostics collected during one compile phase instead of raised.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompilerDiagnostic - - -class CollectedCompileDiagnostics: - """Deduplicated compile diagnostics gathered during one compile phase.""" - - def __init__(self) -> None: - self._by_key: dict[tuple[str, ...], CompilerDiagnostic] = {} - - def add(self, *, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None: - """Record one diagnostic; a repeated key keeps the first diagnostic.""" - - self._by_key.setdefault(key, diagnostic) - - @property - def diagnostics(self) -> tuple[CompilerDiagnostic, ...]: - """Return the recorded diagnostics in deterministic key order.""" - - return tuple(self._by_key[key] for key in sorted(self._by_key)) diff --git a/src/sqlbuild/compiler/compile/classes/compile_input_read_registry.py b/src/sqlbuild/compiler/compile/classes/compile_input_read_registry.py deleted file mode 100644 index 8cb4d1a463..0000000000 --- a/src/sqlbuild/compiler/compile/classes/compile_input_read_registry.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Process-wide delivery of template input reads to the active compile recorders.""" - -from __future__ import annotations - -import threading -from collections.abc import Iterator -from contextlib import contextmanager - -from sqlbuild._native import CompileInputReads -from sqlbuild.compiler.compile.types import CompileContextKey - - -class CompileInputReadRegistry: - """Deliver template reads from every thread, including ones without copied context.""" - - def __init__(self) -> None: - self._lock: threading.Lock = threading.Lock() - self._recorders: tuple[CompileInputReads, ...] = () - - @contextmanager - def recording(self) -> Iterator[CompileInputReads]: - """Record environment variables and run-identity reads until the block exits.""" - - recorder: CompileInputReads = CompileInputReads() - with self._lock: - self._recorders = (*self._recorders, recorder) - try: - yield recorder - finally: - with self._lock: - self._recorders = tuple(item for item in self._recorders if item is not recorder) - - def environment_read(self, name: str) -> None: - """Note that a template looked up one environment variable, present or not.""" - - for recorder in self._recorders: - recorder.add_environment_name(name) - - def settings_class_read(self, settings_class: type) -> None: - """Note a provider settings class instantiated from the environment and files.""" - - for recorder in self._recorders: - recorder.add_settings_class(settings_class) - - def context_read(self, name: str) -> None: - """Note a CTX lookup; only the per-invocation run identity is volatile.""" - - if name != CompileContextKey.RUN_ID: - return - for recorder in self._recorders: - recorder.mark_run_id_read() diff --git a/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py b/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py deleted file mode 100644 index a01db8e479..0000000000 --- a/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py +++ /dev/null @@ -1,19 +0,0 @@ -"""SQL-test CTE extraction class.""" - -from __future__ import annotations - -from sqlbuild._native import extract_sql_test_ctes -from sqlbuild.compiler.compile.models import CompileSqlTestCte -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -class SqlTestCteExtractor: - """Expose authored SQL-test CTE parsing to compiler consumers.""" - - @staticmethod - def extract( - *, sql: str, file_label: str, syntax: SqlLexicalSyntax - ) -> tuple[CompileSqlTestCte, ...]: - """Extract top-level SQL-test CTEs without classifying their roles.""" - - return extract_sql_test_ctes(sql=sql, file_label=file_label, syntax=syntax) diff --git a/src/sqlbuild/compiler/compile/classes/sql_test_cte_locator.py b/src/sqlbuild/compiler/compile/classes/sql_test_cte_locator.py deleted file mode 100644 index 139df54c06..0000000000 --- a/src/sqlbuild/compiler/compile/classes/sql_test_cte_locator.py +++ /dev/null @@ -1,32 +0,0 @@ -"""SQL-test CTE location class.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile -from sqlbuild.spec.contracts.models import SourceLocation - - -class SqlTestCteLocator: - """Expose SQL-test CTE and reference-call locations to compiler consumers.""" - - @staticmethod - def locate( - *, - test_file: DiscoveredSqlTestFile, - test_block: DiscoveredSqlTestBlock, - cte_name: str, - call: str | None, - ) -> SourceLocation: - """Locate a call inside one CTE of a SQL test block, else the CTE header, else the block.""" - - line, column, end_line, end_column = _native.sql_test_cte_span( - test_file.contents, test_block.sql_body, cte_name, call - ) - return SourceLocation( - path=test_file.relative_path, - line=line, - column=column, - end_line=end_line, - end_column=end_column, - ) diff --git a/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py b/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py deleted file mode 100644 index f13c663dde..0000000000 --- a/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py +++ /dev/null @@ -1,44 +0,0 @@ -"""The compile's native SQL-test scan store, opened on first use and saved once.""" - -from __future__ import annotations - -import logging -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity -from sqlbuild.compiler.profiling.main._metric import record_compile_metric - - -class SqlTestScanCache: - """The compile cache folder's scan store, keyed by the installed and native code.""" - - def __init__(self, *, cache_dir: Path | None) -> None: - self._cache_dir: Path | None = cache_dir - self._store: _native.SqlTestScanStore | None = None - - def opened(self) -> _native.SqlTestScanStore | None: - """The native store this compile reads and writes, or None when it has none.""" - - if self._cache_dir is None or self._store is not None: - return self._store - error: str | None - self._store, error = _native.SqlTestScanStore.open_in( - str(self._cache_dir), compiled_code_identity() - ) - if error is not None: - logging.getLogger(__name__).debug("SQL test scan store not loaded: %s", error) - self._cache_dir = None - return self._store - - def save(self) -> None: - """Persist results stored by this compile and report hit and miss counts.""" - - hits, misses = (0, 0) if self._store is None else self._store.counts() - record_compile_metric(metric="sql_test_scan_cache_hits", value=hits) - record_compile_metric(metric="sql_test_scan_cache_misses", value=misses) - if self._store is None: - return - _written, error = self._store.save() - if error is not None: - logging.getLogger(__name__).debug("SQL test scan store not saved: %s", error) diff --git a/src/sqlbuild/compiler/compile/classes/unicode_environment.py b/src/sqlbuild/compiler/compile/classes/unicode_environment.py deleted file mode 100644 index 2b8ef3993c..0000000000 --- a/src/sqlbuild/compiler/compile/classes/unicode_environment.py +++ /dev/null @@ -1,37 +0,0 @@ -"""The process environment as compile text reads it, rejecting values that are not UTF-8.""" - -from __future__ import annotations - -import os -from collections.abc import Iterator, Mapping - -from sqlbuild.compiler.compile.exceptions import CompileInputError - - -class UnicodeEnvironment(Mapping[str, str]): - """`os.environ`, raising a clear error for a value holding undecodable bytes.""" - - def __getitem__(self, name: str) -> str: - value: str = os.environ[name] - try: - _ = value.encode("utf-8") - except UnicodeEncodeError as error: - byte_offset: int = len(value[: error.start].encode("utf-8")) - raise CompileInputError( - f"Environment variable '{name}' is not valid UTF-8 text: byte {byte_offset} " - "starts an invalid UTF-8 sequence", - help=( - f"Set '{name}' to UTF-8 text, since SQLBuild cannot send undecodable bytes " - "to a warehouse; the value is not shown because it may be a secret" - ), - ) from None - return value - - def __contains__(self, name: object) -> bool: - return name in os.environ - - def __iter__(self) -> Iterator[str]: - return iter(os.environ) - - def __len__(self) -> int: - return len(os.environ) diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index dfc9b660ad..50c48ffdaf 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -2,9 +2,6 @@ from __future__ import annotations -from sqlbuild.compiler.compile.classes.compile_input_read_registry import ( - CompileInputReadRegistry, -) from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.python_nodes.types import SqlResourceRefKind @@ -92,7 +89,6 @@ DEFAULT_SQL_TEST_MODE: SqlTestMode = SqlTestMode.MODEL COMPILE_CACHE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_CACHE" COMPILE_CACHE_DISABLE_VALUE: str = "1" -COMPILE_INPUT_READS: CompileInputReadRegistry = CompileInputReadRegistry() RETIRED_FACT_CACHE_DIRECTORY_NAME: str = "facts-v1" RETIRED_REFERENCE_CACHE_DIRECTORY_NAME: str = "references-v3" SQL_TEST_CTE_SCAN_ALGORITHM: str = "expanded-sql-test-ctes-v1" diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index d219e66130..b0a97c1a9d 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -10,9 +10,9 @@ from typing import Any, overload import sqlbuild._native as _native +from sqlbuild._native import SqlTestScanCache from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity -from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE, MACRO_CONTEXT_PARAMETER_NAME from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import ( diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index 01c2391b5d..38ba355622 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -614,19 +614,6 @@ def resource_type(self) -> CompiledResourceType: return CompiledResourceType(self.value) -class CompileContextKey(StrEnum): - RUN_ID = "run.id" - RUN_TARGET = "run.target" - MODEL_NAME = "model.name" - MODEL_DATABASE = "model.database" - MODEL_SCHEMA = "model.schema" - MODEL_ALIAS = "model.alias" - DESTINATION_DATABASE = "destination.database" - DESTINATION_SCHEMA = "destination.schema" - DESTINATION_TABLE = "destination.table" - DESTINATION_QUALIFIED = "destination.qualified" - - class CompiledResourceType(StrEnum): MODEL = "model" SOURCE = "source" diff --git a/src/sqlbuild/compiler/contract_adoption/main/__init__.py b/src/sqlbuild/compiler/contract_adoption/main/__init__.py deleted file mode 100644 index 443652aac9..0000000000 --- a/src/sqlbuild/compiler/contract_adoption/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Public contract adoption operations.""" diff --git a/src/sqlbuild/compiler/contract_adoption/main/_write.py b/src/sqlbuild/compiler/contract_adoption/main/_write.py deleted file mode 100644 index 7050ced037..0000000000 --- a/src/sqlbuild/compiler/contract_adoption/main/_write.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Typed, metadata-preserving repository edits for contract adoption.""" - -from __future__ import annotations - -from dataclasses import replace -from pathlib import Path - -from sqlbuild._native import ( - build_compiled_project, - discover_project_inputs, - edit_contract_model, - edit_contract_source, - remaining_contract_findings, - write_atomically, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( - CompileAnalysisSelection, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledResourceType, - CompiledSourceView, -) -from sqlbuild.compiler.contract_adoption.models import ( - ContractAdoptionResult, - ContractEvidence, - ContractFinding, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ProjectGraph - - -def write_contracts( - *, - project_dir: Path, - graph: ProjectGraph, - result: ContractAdoptionResult, - overwrite: bool, - adapter: BaseAdapter | None = None, - resolved_connection: dict[str, object] | None = None, - cli_vars: dict[str, object] | None = None, -) -> ContractAdoptionResult: - """Apply safe source-only edits atomically and validate the resulting project.""" - - models: dict[str, CompiledModelView] = {model.name: model for model in graph.project.models} - sources: dict[str, CompiledSourceView] = { - source.name: source for source in graph.project.sources - } - updates: dict[Path, str] = {} - updated_evidence: list[ContractEvidence] = [] - for evidence in result.evidence: - if evidence.physical_columns is None or not evidence.findings: - updated_evidence.append(evidence) - continue - if evidence.resource_type == CompiledResourceType.MODEL: - contents, conflict = edit_contract_model( - model=models[evidence.resource_name], - physical_columns=evidence.physical_columns, - overwrite=overwrite, - ) - path: Path = project_dir / models[evidence.resource_name].relative_path - else: - path = sources[evidence.resource_name].source_file.file_path - contents, conflict = edit_contract_source( - source=sources[evidence.resource_name], - physical_columns=evidence.physical_columns, - overwrite=overwrite, - contents=updates.get(path), - ) - if contents is not None and contents != path.read_text(encoding="utf-8"): - updates[path] = contents - remaining: tuple[ContractFinding, ...] = remaining_contract_findings( - findings=evidence.findings, - overwrite=overwrite, - conflict=conflict, - ) - updated_evidence.append(replace(evidence, findings=remaining)) - if not updates: - return replace(result, evidence=tuple(updated_evidence)) - originals: dict[Path, str] = {path: path.read_text(encoding="utf-8") for path in updates} - try: - for path, contents in updates.items(): - write_atomically(path=path, contents=contents) - discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) - if adapter is not None: - _ = build_compiled_project( - discovered_inputs=discovered, - adapter=adapter, - selected_target=result.from_target, - resolved_connection=resolved_connection, - cli_vars=cli_vars, - analysis_selection=CompileAnalysisSelection(no_cache=True), - ) - except Exception: - for path, contents in originals.items(): - write_atomically(path=path, contents=contents) - raise - return replace( - result, - evidence=tuple(updated_evidence), - written_paths=tuple(sorted(path.relative_to(project_dir) for path in updates)), - ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index c981bd9dc3..89d26cad35 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -14,10 +14,10 @@ from typing import TYPE_CHECKING, Any, get_type_hints import sqlbuild._native as _native -from sqlbuild._native import validate_resource_identity -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS -from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import ( +from sqlbuild._native import ( project_relative_path, + record_settings_class_read, + validate_resource_identity, ) from sqlbuild.compiler.discovery.constants import ( PYTHON_INIT_MODULE_STEM, @@ -216,7 +216,7 @@ def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, ) = cached_event_exporter_extensions(project_dir=project_dir) if cached is not None: for cached_provider in cached[0]: - COMPILE_INPUT_READS.settings_class_read(cached_provider.provider_class) + record_settings_class_read(cached_provider.provider_class) return cached[0] providers_root: Path = project_dir / "providers" if not providers_root.is_dir(): @@ -249,7 +249,7 @@ def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, f"{file_path.relative_to(project_dir)}" ) seen_names[provider_name] = file_path - COMPILE_INPUT_READS.settings_class_read(provider_class) + record_settings_class_read(provider_class) discovered_providers.append( DiscoveredProvider( file_path=file_path, diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py deleted file mode 100644 index 0757686bc3..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Project-relative path facts shared by filesystem discovery.""" - -from __future__ import annotations - -from pathlib import Path - - -def project_relative_path(*, path: Path, project_dir: Path) -> Path: - """Return path.relative_to(project_dir) without pathlib's per-parent comparison scan.""" - - root_parts: tuple[str, ...] = project_dir.parts - path_parts: tuple[str, ...] = path.parts - if ( - path.is_absolute() is not project_dir.is_absolute() - or path_parts[: len(root_parts)] != root_parts - ): - return path.relative_to(project_dir) - return path.with_segments(*path_parts[len(root_parts) :]) diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/yaml_classes.py b/src/sqlbuild/compiler/discovery/_helpers/yml/yaml_classes.py deleted file mode 100644 index 2b96c6cf9f..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/yaml_classes.py +++ /dev/null @@ -1,45 +0,0 @@ -"""The Python classes native YAML entry parsing builds source and seed declarations as.""" - -from __future__ import annotations - -from functools import cache - -from sqlbuild.compiler.model_config.main._native_contract_classes import native_contract_classes -from sqlbuild.integrations.dlt.models import DltResourceConfig, DltSourceConfig -from sqlbuild.integrations.ingestr.models import IngestrSourceConfig -from sqlbuild.spec.contracts.models import ( - IntegrationLoaderConfig, - SchemaSeedEntry, - SeedCsvSettings, - SourceColumnEntry, - SourceEntry, - SourceFreshnessAgePolicy, - SourceFreshnessConfig, -) -from sqlbuild.spec.contracts.types import ( - SourceFreshnessStrategy, - SourceFreshnessValueKind, - SourceWriteStrategy, -) - - -@cache -def yaml_entry_classes() -> dict[str, object]: - """The classes parsed seed and source declarations become.""" - - return { - "contracts": native_contract_classes(), - "schema_seed_entry": SchemaSeedEntry, - "seed_csv_settings": SeedCsvSettings, - "source_entry": SourceEntry, - "source_column_entry": SourceColumnEntry, - "source_freshness_config": SourceFreshnessConfig, - "source_freshness_age_policy": SourceFreshnessAgePolicy, - "integration_loader_config": IntegrationLoaderConfig, - "ingestr_source_config": IngestrSourceConfig, - "dlt_source_config": DltSourceConfig, - "dlt_resource_config": DltResourceConfig, - "source_write_strategy": SourceWriteStrategy, - "source_freshness_strategy": SourceFreshnessStrategy, - "source_freshness_value_kind": SourceFreshnessValueKind, - } diff --git a/src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py b/src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py deleted file mode 100644 index 30fa09e4a7..0000000000 --- a/src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Locate one named entry in an authored YAML declaration file.""" - -from __future__ import annotations - -from collections.abc import Mapping -from functools import lru_cache - -import sqlbuild._native as _native - - -def yaml_entry_line(*, contents: str, name: str) -> int | None: - """Return the 1-based line of the outermost `- name: ` entry, if present.""" - - return _entry_lines(contents).get(name) - - -@lru_cache(maxsize=16) -def _entry_lines(contents: str) -> Mapping[str, int]: - return dict(_native.yaml_entry_line_index(contents)) diff --git a/src/sqlbuild/compiler/frontier/main/compiler_cache_directory.py b/src/sqlbuild/compiler/frontier/main/compiler_cache_directory.py deleted file mode 100644 index cf99e3f7e0..0000000000 --- a/src/sqlbuild/compiler/frontier/main/compiler_cache_directory.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Project-local compiler store root.""" - -from pathlib import Path - -from sqlbuild.compiler.frontier.constants import ( - CACHE_DIRECTORY_NAME, - COMPILER_CACHE_DIRECTORY_NAME, - NATIVE_CACHE_NAMESPACE_SUFFIX, - TARGET_DIRECTORY_NAME, -) - - -def compiler_cache_directory(project_dir: Path) -> Path: - """Return the project's compiler store root.""" - - return ( - project_dir - / TARGET_DIRECTORY_NAME - / CACHE_DIRECTORY_NAME - / f"{COMPILER_CACHE_DIRECTORY_NAME}{NATIVE_CACHE_NAMESPACE_SUFFIX}" - ) diff --git a/src/sqlbuild/compiler/model_config/__init__.py b/src/sqlbuild/compiler/model_config/__init__.py deleted file mode 100644 index cdb238defa..0000000000 --- a/src/sqlbuild/compiler/model_config/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native model configuration.""" diff --git a/src/sqlbuild/compiler/model_config/constants.py b/src/sqlbuild/compiler/model_config/constants.py deleted file mode 100644 index ff4deb0c27..0000000000 --- a/src/sqlbuild/compiler/model_config/constants.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Names and error classes the native model config bindings share.""" - -from __future__ import annotations - -ENVIRONMENT_READ: str = "env" -CONFIG_VALUE_TYPE_ERROR: str = "config_value_type" -RESOURCE_IDENTITY_ERROR: str = "resource_identity" -DISCOVERY_CONFLICT_ERROR: str = "discovery_conflict" -EXPECTED_STRING: str = "a string" diff --git a/src/sqlbuild/compiler/model_config/main/__init__.py b/src/sqlbuild/compiler/model_config/main/__init__.py deleted file mode 100644 index 8665a5bb71..0000000000 --- a/src/sqlbuild/compiler/model_config/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native model configuration entry points.""" diff --git a/src/sqlbuild/compiler/model_config/main/_native_config_error.py b/src/sqlbuild/compiler/model_config/main/_native_config_error.py deleted file mode 100644 index b83dab89d6..0000000000 --- a/src/sqlbuild/compiler/model_config/main/_native_config_error.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Raise a native config stage's exact error as the Python stage raises it.""" - -from __future__ import annotations - -from collections.abc import Mapping - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.discovery.exceptions import DiscoveryConflictError -from sqlbuild.compiler.model_config.constants import ( - CONFIG_VALUE_TYPE_ERROR, - DISCOVERY_CONFLICT_ERROR, - EXPECTED_STRING, - RESOURCE_IDENTITY_ERROR, -) -from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError -from sqlbuild.spec.contracts.exceptions import ConfigValueTypeError - - -def native_config_error( - *, - error: _native.NativeConfigError, - values: Mapping[str, object] | None = None, -) -> Exception: - """Return the exception class `error` names, with its message, code and help.""" - - if error.class_name == CONFIG_VALUE_TYPE_ERROR and error.key is not None and values is not None: - return ConfigValueTypeError( - key=error.key, expected=EXPECTED_STRING, actual_type=type(values[error.key]) - ) - if error.class_name == RESOURCE_IDENTITY_ERROR: - return ResourceIdentityError(error.message, help=error.help) - if error.class_name == DISCOVERY_CONFLICT_ERROR: - return DiscoveryConflictError(error.message, help=error.help) - return CompileInputError(error.message, code=error.code, help=error.help) diff --git a/src/sqlbuild/compiler/model_config/main/_native_contract_classes.py b/src/sqlbuild/compiler/model_config/main/_native_contract_classes.py deleted file mode 100644 index ee045084c6..0000000000 --- a/src/sqlbuild/compiler/model_config/main/_native_contract_classes.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Contract classes native header parsing builds Python columns and audits from.""" - -from functools import cache - -from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds -from sqlbuild.compiler.auditing.types import AuditSeverity, ThresholdOperator -from sqlbuild.spec.contracts.models import ( - SchemaAuditInstance, - SchemaColumn, - SchemaDynamicColumnFamily, - SourceLocation, -) - - -@cache -def native_contract_classes() -> dict[str, object]: - """The contract classes native header parsing builds columns and audits as.""" - - return { - "schema_column": SchemaColumn, - "schema_audit_instance": SchemaAuditInstance, - "schema_dynamic_column_family": SchemaDynamicColumnFamily, - "source_location": SourceLocation, - "measurement_thresholds": MeasurementThresholds, - "measurement_threshold_bound": MeasurementThresholdBound, - "severities": {severity.value: severity for severity in AuditSeverity}, - "threshold_operators": {operator.value: operator for operator in ThresholdOperator}, - "allocate": object.__new__, - } diff --git a/src/sqlbuild/compiler/path_defaults/__init__.py b/src/sqlbuild/compiler/path_defaults/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/path_defaults/_helpers/__init__.py b/src/sqlbuild/compiler/path_defaults/_helpers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/path_defaults/_helpers/matching.py b/src/sqlbuild/compiler/path_defaults/_helpers/matching.py deleted file mode 100644 index f7caf920e1..0000000000 --- a/src/sqlbuild/compiler/path_defaults/_helpers/matching.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Path-default glob matching mechanics.""" - -from __future__ import annotations - -from functools import cache - -from sqlbuild.compiler.path_defaults.constants import ( - GLOB_SEGMENTS, - RECURSIVE_SEGMENT_GLOB, - SINGLE_SEGMENT_GLOB, -) - - -def matches_prefix(*, path_parts: tuple[str, ...], path_key: str) -> bool: - """Return whether a path-default pattern matches a prefix of a model path.""" - - pattern_parts: tuple[str, ...] = tuple(path_key.split("/")) - - @cache - def matches(*, pattern_index: int, path_index: int) -> bool: - if pattern_index == len(pattern_parts): - return True - pattern_part: str = pattern_parts[pattern_index] - if pattern_part == RECURSIVE_SEGMENT_GLOB: - return matches(pattern_index=pattern_index + 1, path_index=path_index) or ( - path_index < len(path_parts) - and matches(pattern_index=pattern_index, path_index=path_index + 1) - ) - if path_index == len(path_parts): - return False - if pattern_part == SINGLE_SEGMENT_GLOB or pattern_part == path_parts[path_index]: - return matches(pattern_index=pattern_index + 1, path_index=path_index + 1) - return False - - return matches(pattern_index=0, path_index=0) - - -def is_wildcard(*, path_key: str) -> bool: - """Return whether a path-default key contains a supported wildcard segment.""" - - return any(part in GLOB_SEGMENTS for part in path_key.split("/")) - - -def specificity(*, path_key: str) -> tuple[int, int, int]: - """Return deterministic wildcard specificity based on literal path structure.""" - - parts: tuple[str, ...] = tuple(path_key.split("/")) - literal_count: int = sum(part not in GLOB_SEGMENTS for part in parts) - single_segment_count: int = parts.count(SINGLE_SEGMENT_GLOB) - recursive_count: int = parts.count(RECURSIVE_SEGMENT_GLOB) - return literal_count, single_segment_count, -recursive_count diff --git a/src/sqlbuild/compiler/path_defaults/constants.py b/src/sqlbuild/compiler/path_defaults/constants.py deleted file mode 100644 index 094f5cba2c..0000000000 --- a/src/sqlbuild/compiler/path_defaults/constants.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Constants for path-default glob matching.""" - -from __future__ import annotations - -SINGLE_SEGMENT_GLOB: str = "*" -RECURSIVE_SEGMENT_GLOB: str = "**" -GLOB_SEGMENTS: frozenset[str] = frozenset({SINGLE_SEGMENT_GLOB, RECURSIVE_SEGMENT_GLOB}) -UNSUPPORTED_GLOB_MARKERS: tuple[str, ...] = (SINGLE_SEGMENT_GLOB, "?", "[") diff --git a/src/sqlbuild/compiler/path_defaults/main/__init__.py b/src/sqlbuild/compiler/path_defaults/main/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/path_defaults/main/_select.py b/src/sqlbuild/compiler/path_defaults/main/_select.py deleted file mode 100644 index 7970f191e3..0000000000 --- a/src/sqlbuild/compiler/path_defaults/main/_select.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Shared path-default matching and selection entrypoint.""" - -from __future__ import annotations - -from functools import lru_cache - -from sqlbuild.compiler.discovery.exceptions import DiscoveryConflictError -from sqlbuild.compiler.path_defaults._helpers.matching import ( - is_wildcard, - matches_prefix, - specificity, -) -from sqlbuild.compiler.path_defaults.models import PathDefaultSelection - - -def select_path_default(*, model_path: str, path_keys: tuple[str, ...]) -> PathDefaultSelection: - """Match and deterministically select a path default for a model path.""" - - normalized_path: str = model_path.replace("\\", "/").removeprefix("models/") - return _select_normalized_path_default(normalized_path=normalized_path, path_keys=path_keys) - - -@lru_cache(maxsize=32_768) -def _select_normalized_path_default( - *, normalized_path: str, path_keys: tuple[str, ...] -) -> PathDefaultSelection: - path_parts: tuple[str, ...] = tuple(part for part in normalized_path.split("/") if part) - matched_keys: tuple[str, ...] = tuple( - sorted( - path_key - for path_key in path_keys - if matches_prefix(path_parts=path_parts, path_key=path_key) - ) - ) - literal_matches: tuple[str, ...] = tuple( - path_key for path_key in matched_keys if not is_wildcard(path_key=path_key) - ) - if literal_matches: - return PathDefaultSelection( - matched_keys=matched_keys, - selected_key=max(literal_matches, key=lambda path_key: len(path_key.split("/"))), - ) - if not matched_keys: - return PathDefaultSelection(matched_keys=(), selected_key=None) - - best_score: tuple[int, int, int] = max( - specificity(path_key=path_key) for path_key in matched_keys - ) - best_matches: tuple[str, ...] = tuple( - path_key for path_key in matched_keys if specificity(path_key=path_key) == best_score - ) - if len(best_matches) > 1: - conflicting_keys: str = ", ".join(f"'{path_key}'" for path_key in best_matches) - raise DiscoveryConflictError( - f"Model path '{normalized_path}' matches equally specific path_defaults keys: " - f"{conflicting_keys}.", - help=( - "Make one pattern narrower by adding literal or '*' path segments, or remove the " - "overlap. Path-default selection never depends on declaration order." - ), - ) - return PathDefaultSelection(matched_keys=matched_keys, selected_key=best_matches[0]) diff --git a/src/sqlbuild/compiler/path_defaults/models.py b/src/sqlbuild/compiler/path_defaults/models.py deleted file mode 100644 index a97bce0052..0000000000 --- a/src/sqlbuild/compiler/path_defaults/models.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Structured path-default selection models.""" - -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class PathDefaultSelection: - """Deterministic path-default matching result for one model path.""" - - matched_keys: tuple[str, ...] - selected_key: str | None diff --git a/src/sqlbuild/compiler/pipeline/main/__init__.py b/src/sqlbuild/compiler/pipeline/main/__init__.py deleted file mode 100644 index bd9b94f6a1..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Pipeline main package.""" diff --git a/src/sqlbuild/compiler/pipeline/main/_plan_result.py b/src/sqlbuild/compiler/pipeline/main/_plan_result.py deleted file mode 100644 index 3da6c17930..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/_plan_result.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Plan a compiled project over an open warehouse connection.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.materializations import load_custom_materializations -from sqlbuild.compiler.pipeline.models import ( - CompilePipelineOptions, - CompilePipelineResult, - PythonRunPlanOutputs, -) -from sqlbuild.compiler.planner.main.execution.execution import build_execution_plan -from sqlbuild.compiler.planner.models import ( - DeferralInputs, - PlannerOverrides, - PlannerPolicies, - PlannerSelection, - PlanOutput, -) -from sqlbuild.compiler.python_nodes.main._run_selection import ( - resolve_python_sql_run_selection_from_inputs, -) -from sqlbuild.compiler.python_nodes.models import PythonSqlRunSelection -from sqlbuild.spec.contracts.models import TargetConfig - - -def build_pipeline_result( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - connection: Any, - project: CompiledProjectView, - options: CompilePipelineOptions, - on_progress: Callable[[str], None] | None = None, -) -> CompilePipelineResult: - """Plan the compiled project over an open connection and attach Python node plans.""" - - select: tuple[str, ...] = options.select - exclude: tuple[str, ...] = options.exclude - deferred_locations: dict[str, CompiledRelationLocation] | None = None - deferred_relations: dict[str, RelationInfo] | None = None - if options.defer_to is not None: - deferred_target_config: TargetConfig = _native.resolve_deferred_target_config( - discovered_inputs=discovered_inputs, - defer_to=options.defer_to, - current_target_name=project.effective_target_name, - ) - deferred_locations = _native.build_deferred_locations( - project=project, - deferred_target_config=deferred_target_config, - effective_vars=project.effective_vars, - default_schema=adapter.default_schema(), - default_database=adapter.default_database(), - render_qualified_name=adapter.render_qualified_name, - ) - deferred_relations = _native.gather_deferred_relations( - adapter=adapter, - connection=connection, - deferred_locations=deferred_locations, - ) - - selected_sql_keys: frozenset[CompiledObjectKey] | None = None - selected_python_node_names: frozenset[str] = frozenset() - python_read_source_names: frozenset[str] = frozenset() - run_selection: PythonSqlRunSelection | None = None - if not options.sql_test_selection.models_selected: - selected_sql_keys = frozenset() - elif options.resolve_python_run_selectors: - run_selection = resolve_python_sql_run_selection_from_inputs( - select=select, - exclude=exclude, - project_graph=_native.build_project_graph_from_compiled_project(project=project), - discovered_inputs=discovered_inputs, - ) - selected_sql_keys = run_selection.sql_keys - selected_python_node_names = run_selection.python_node_names - python_read_source_names = _native.python_read_source_names_for( - discovered_inputs=discovered_inputs, python_node_names=selected_python_node_names - ) - - plan_output: PlanOutput = build_execution_plan( - project=project, - adapter=adapter, - connection=connection, - selection=PlannerSelection( - select=select, - exclude=exclude, - selected_keys=selected_sql_keys, - python_read_source_names=python_read_source_names, - sql_test_selection=options.sql_test_selection, - ), - overrides=PlannerOverrides( - cursor_overrides=options.cursor_overrides, - full_refresh=options.full_refresh, - reload_sources=options.reload_sources, - max_microbatches=options.max_microbatches, - ), - deferral=DeferralInputs( - deferred_locations=deferred_locations, - deferred_relations=deferred_relations, - defer_sources_to=options.defer_sources_to, - source_deferral_enabled=options.source_deferral_enabled, - ), - policies=PlannerPolicies( - auto_load_sources=options.auto_load_sources, - selection_diagnostics=options.selection_diagnostics, - plan_sql_tests=options.plan_sql_tests, - record_migration_fingerprints=options.record_migration_fingerprints, - ), - on_progress=on_progress, - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - custom_materializations: dict[str, Any] = load_custom_materializations( - discovered_inputs.materialization_files - ) - python_outputs: PythonRunPlanOutputs = _native.build_python_run_plan_outputs( - discovered_inputs=discovered_inputs, - plan_output=plan_output, - run_selection=run_selection, - selected_python_node_names=selected_python_node_names, - ) - - return CompilePipelineResult( - project=project, - plan_output=python_outputs.plan_output, - custom_materializations=custom_materializations, - python_node_names=python_outputs.selected_python_node_names, - python_plan_entries=python_outputs.python_plan_entries, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/__init__.py b/src/sqlbuild/compiler/planner/_helpers/__init__.py deleted file mode 100644 index 3ea78f03da..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Planner helper utilities.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/__init__.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/__init__.py deleted file mode 100644 index 8dc4d94372..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Internal focused-command planning phases.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/seed_state.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/seed_state.py deleted file mode 100644 index 650547e835..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/seed_state.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Targeted warehouse identity state for direct seed commands.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild._native import read_latest_fingerprints -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.types import CompiledSeedView -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME, NODE_TYPE_SEED -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet - - -def read_seed_fingerprints( - *, adapter: BaseAdapter, connection: Any, seeds: tuple[CompiledSeedView, ...] -) -> dict[str, Fingerprint]: - """Read only fingerprint state required to classify selected direct seeds.""" - - names_by_target: dict[tuple[str | None, str], set[str]] = {} - for seed in seeds: - if seed.destination.schema is None: - continue - names_by_target.setdefault((seed.destination.database, seed.destination.schema), set()).add( - seed.name - ) - - targets: tuple[tuple[str | None, str], ...] = tuple( - sorted(names_by_target, key=lambda target: ((target[0] or ""), target[1])) - ) - relations_by_target: dict[tuple[str | None, str], tuple[RelationInfo, ...]] = ( - _gather_fingerprint_relations( - adapter=adapter, - connection=connection, - targets=targets, - ) - ) - fingerprints: dict[str, Fingerprint] = {} - for database, schema in targets: - names: set[str] = names_by_target[(database, schema)] - relations: tuple[RelationInfo, ...] = relations_by_target[(database, schema)] - table_exists: bool = any( - relation.name.lower() == FINGERPRINT_TABLE_NAME.lower() for relation in relations - ) - fingerprint_set: FingerprintSet = read_latest_fingerprints( - connection=connection, - execute=adapter.execute, - table_exists=table_exists, - database=database, - schema=schema, - render_qualified_name=adapter.render_qualified_name, - render_read_latest_sql=adapter.render_read_latest_fingerprints_sql, - node_names=tuple(sorted(names)), - ) - fingerprints.update( - { - name: fingerprint - for name, fingerprint in fingerprint_set.fingerprints.items() - if fingerprint.node_type == NODE_TYPE_SEED and name in names - } - ) - return fingerprints - - -def _gather_fingerprint_relations( - *, - adapter: BaseAdapter, - connection: Any, - targets: tuple[tuple[str | None, str], ...], -) -> dict[tuple[str | None, str], tuple[RelationInfo, ...]]: - if not targets: - return {} - database, schema = targets[0] - relations: tuple[RelationInfo, ...] = adapter.list_relations( - connection=connection, - database=database, - schemas=(schema,), - names=(FINGERPRINT_TABLE_NAME,), - ) - return { - (database, schema): relations, - **_gather_fingerprint_relations( - adapter=adapter, - connection=connection, - targets=targets[1:], - ), - } diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py deleted file mode 100644 index 06b82aa8e6..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Canonical static planning phases for commands that do not build models.""" - -from __future__ import annotations - -from sqlbuild._native import ( - build_compiled_function_fingerprint_sql, - build_seed_identity, - build_selected_audit_entries, - extract_seed_columns, - get_materialization_type, - resolve_planner_scopes, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context -from sqlbuild.compiler.planner._helpers.output.plan_output import ( - build_selected_test_entries, - plan_function, - sql_test_is_selected, -) -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - DeferralInputs, - FunctionPlanEntry, - PlannerPolicies, - PlannerRelationsContext, - PlannerScope, - PlannerSelection, - PlanOutput, - PlanWarning, - SeedPlanEntry, - SqlTestPlanEntry, -) -from sqlbuild.compiler.planner.types import PlanReason -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -def resolve_static_command_scope_impl( - *, - project: CompiledProjectView, - selection: PlannerSelection, - auto_load_sources: bool = False, -) -> PlannerScope: - """Resolve command selection through the same canonical selector phase as builds.""" - - return resolve_planner_scopes( - project=project, - selection=selection, - policies=PlannerPolicies(auto_load_sources=auto_load_sources), - ).selected_scope - - -def resolve_static_relation_context_impl( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - deferral: DeferralInputs | None = None, - project_config: ProjectConfig | None = None, - local_config: LocalConfig | None = None, - relation_keys: frozenset[CompiledObjectKey] | None = None, -) -> PlannerRelationsContext: - """Resolve canonical locations and source routing without warehouse inventory.""" - - return build_planner_relations_context( - project=project, - adapter=adapter, - connection=None, - scope=scope, - deferral=deferral, - project_config=project_config, - local_config=local_config, - known_source_columns={}, - relation_keys=relation_keys, - ) - - -def build_audit_command_plan_impl( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - relations: PlannerRelationsContext, -) -> PlanOutput: - """Project the selected audits from static compile and relation state.""" - - model_materializations: dict[str, str] = { - model.name: get_materialization_type(model).value for model in project.models - } - return _base_plan_output( - project=project, - scope=scope, - relations=relations, - audit_entries=tuple( - build_selected_audit_entries( - project=project, - adapter=adapter, - scope=scope, - relations=relations, - model_materializations=model_materializations, - ) - ), - ) - - -def build_test_command_plan_impl( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - relations: PlannerRelationsContext, - case_name: str | None = None, -) -> PlanOutput: - """Project selected SQL tests from static compile state.""" - - test_entries, warnings = build_selected_test_entries( - project=project, - adapter=adapter, - selected_keys=scope.selected_keys, - sql_test_selection=scope.sql_test_selection, - case_name=case_name, - ) - available_test_case_names: tuple[str, ...] = tuple( - sorted( - { - test.case_name - for test in project.sql_tests - if test.case_name is not None - and sql_test_is_selected( - test=test, - selected_keys=scope.selected_keys, - sql_test_selection=scope.sql_test_selection, - ) - } - ) - ) - test_keys: frozenset[CompiledObjectKey] = frozenset(entry.key for entry in test_entries) - function_keys_set: set[CompiledObjectKey] = set() - for test_entry in test_entries: - function_keys_set.update(test_entry.function_deps) - function_keys: frozenset[CompiledObjectKey] = frozenset(function_keys_set) - function_entries: tuple[FunctionPlanEntry, ...] = tuple( - plan_function( - function=function, - adapter=adapter, - relations=relations, - fingerprint_query_sql=build_compiled_function_fingerprint_sql(function), - ) - for function in project.functions - if function.key in function_keys - ) - return _base_plan_output( - project=project, - scope=scope, - relations=relations, - execution_order=tuple( - key for key in scope.execution_order if key in scope.selected_keys | test_keys - ), - test_entries=tuple(test_entries), - available_test_case_names=available_test_case_names, - function_entries=function_entries, - warnings=tuple(warnings), - ) - - -def build_seed_command_plan_impl( - *, - project: CompiledProjectView, - scope: PlannerScope, - relations: PlannerRelationsContext, - fingerprints: dict[str, Fingerprint] | None = None, -) -> PlanOutput: - """Project selected direct seed work and canonical identities.""" - - existing: dict[str, Fingerprint] = fingerprints or {} - entries: list[SeedPlanEntry] = [] - for seed in project.seeds: - if seed.key not in scope.selected_keys: - continue - version_hash, metadata_json = build_seed_identity(seed) - previous: Fingerprint | None = existing.get(seed.name) - reason: PlanReason = ( - PlanReason.FIRST_RUN - if previous is None - else PlanReason.CONFIG_CHANGED - if previous.version_hash != version_hash - else PlanReason.NO_CHANGE - ) - entries.append( - SeedPlanEntry( - key=seed.key, - name=seed.name, - destination=seed.destination, - file_path=seed.seed_file.file_path, - columns=extract_seed_columns(seed), - csv_settings=seed.schema_entry.csv_settings, - fingerprint_definition=metadata_json, - fingerprint_version_hash=version_hash, - fingerprint_metadata_json=metadata_json, - reason=reason, - ) - ) - return _base_plan_output( - project=project, - scope=scope, - relations=relations, - seed_entries=tuple(entries), - ) - - -def build_relation_command_plan_impl( - *, project: CompiledProjectView, scope: PlannerScope, relations: PlannerRelationsContext -) -> PlanOutput: - """Project only static relation data for Python checks and scenarios.""" - - return _base_plan_output(project=project, scope=scope, relations=relations) - - -def _base_plan_output( - *, - project: CompiledProjectView, - scope: PlannerScope, - relations: PlannerRelationsContext, - execution_order: tuple[CompiledObjectKey, ...] | None = None, - seed_entries: tuple[SeedPlanEntry, ...] = (), - function_entries: tuple[FunctionPlanEntry, ...] = (), - audit_entries: tuple[AuditPlanEntry, ...] = (), - test_entries: tuple[SqlTestPlanEntry, ...] = (), - available_test_case_names: tuple[str, ...] = (), - warnings: tuple[PlanWarning, ...] = (), -) -> PlanOutput: - return PlanOutput( - execution_order=( - execution_order - if execution_order is not None - else tuple(key for key in scope.execution_order if key in scope.selected_keys) - ), - seed_entries=seed_entries, - function_entries=function_entries, - audit_entries=audit_entries, - test_entries=test_entries, - available_test_case_names=available_test_case_names, - selected_keys=scope.selected_keys, - warnings=warnings, - upstream_deps=scope.upstream_deps, - downstream_deps=scope.downstream_deps, - model_locations=relations.model_locations, - seed_locations=relations.seed_locations, - function_locations=relations.function_locations, - source_map=relations.source_map, - source_read_map=relations.source_read_map, - python_source_read_map=relations.python_source_read_map, - hook_functions=project.hook_functions, - enforce_explicit_references=project.enforce_explicit_references, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/fixtures/__init__.py b/src/sqlbuild/compiler/planner/_helpers/fixtures/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/_helpers/fixtures/empty_fixture.py b/src/sqlbuild/compiler/planner/_helpers/fixtures/empty_fixture.py deleted file mode 100644 index e5f142cbe5..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/fixtures/empty_fixture.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Recognition for the planner's empty relation fixture marker.""" - -from sqlbuild._native import is_empty_fixture_query as is_empty_fixture_query diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/defer_clone.py b/src/sqlbuild/compiler/planner/_helpers/graph/defer_clone.py deleted file mode 100644 index ed367376f2..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/defer_clone.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Deferred clone graph traversal implementations.""" - -from __future__ import annotations - -from collections.abc import Callable, Mapping - - -def resolve_clone_boundary_impl[K]( - *, - selected: frozenset[K], - upstream: Mapping[K, tuple[K, ...]], - is_clonable: Callable[[K], bool], - is_view: Callable[[K], bool], -) -> frozenset[K]: - """Return non-view clonable ancestors anchoring selected nodes.""" - - boundary: frozenset[K] = frozenset() - visited: frozenset[K] = frozenset() - - def visit( - *, node: K, visited: frozenset[K], boundary: frozenset[K] - ) -> tuple[frozenset[K], frozenset[K]]: - if node in visited: - return visited, boundary - visited = visited | {node} - upstream_node: K - for upstream_node in upstream.get(node, ()): - if upstream_node in selected or is_view(upstream_node): - visited, boundary = visit(node=upstream_node, visited=visited, boundary=boundary) - elif is_clonable(upstream_node): - boundary = boundary | {upstream_node} - return visited, boundary - - selected_node: K - for selected_node in selected: - visited, boundary = visit(node=selected_node, visited=visited, boundary=boundary) - return boundary - - -def resolve_skipped_view_chain_impl[K]( - *, - selected: frozenset[K], - upstream: Mapping[K, tuple[K, ...]], - is_clonable: Callable[[K], bool], - is_view: Callable[[K], bool], -) -> frozenset[K]: - """Return skipped view ancestors rebuilt over cloned boundaries.""" - - views: frozenset[K] = frozenset() - visited: frozenset[K] = frozenset() - - def visit( - *, node: K, visited: frozenset[K], views: frozenset[K] - ) -> tuple[frozenset[K], frozenset[K]]: - if node in visited: - return visited, views - visited = visited | {node} - upstream_node: K - for upstream_node in upstream.get(node, ()): - if upstream_node in selected: - visited, views = visit(node=upstream_node, visited=visited, views=views) - elif is_view(upstream_node): - if is_clonable(upstream_node): - views = views | {upstream_node} - visited, views = visit(node=upstream_node, visited=visited, views=views) - return visited, views - - selected_node: K - for selected_node in selected: - visited, views = visit(node=selected_node, visited=visited, views=views) - return views diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/__init__.py b/src/sqlbuild/compiler/planner/_helpers/migrations/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py deleted file mode 100644 index 6dac3632fa..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ /dev/null @@ -1,1101 +0,0 @@ -"""Per-model plan entry construction helpers.""" - -from __future__ import annotations - -from dataclasses import dataclass, replace -from datetime import UTC, datetime -from decimal import Decimal -from typing import Any - -from sqlbuild._native import ( - apply_configured_microbatch_limits, - assemble_model_plan_entry, - audit_is_selected, - build_cursor_input_relations, - build_source_load_map, - build_source_read_map, - check_full_rebuild_allowed, - describe_full_rebuild_cause, - detect_model_changes, - gather_source_inspection, - get_config_cursor_bound, - get_config_str, - get_materialization_type, - get_semantic_source_columns, - microbatch_guidance, - replaces_incremental_relation, - resolve_bounded_cursor_override, - resolve_microbatch_lookback, - resolve_model_full_refresh, - resolve_model_plan_action, - runtime_cursor_producer_names, - source_binding_diagnostics, - validate_capped_watermark_inputs, - validate_source_cursor_input_columns, - with_declared_source_reads, -) -from sqlbuild._native import cursor_render as render -from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, - CompilerDiagnostic, -) -from sqlbuild.compiler.compile.types import ( - CompiledAuditView, - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - compute_cursor_bounds, - cursor_inputs_without_rows, - normalize_cursor_snapshot_grain, - resolve_effective_timestamp_grain, - without_destination_cursor, -) -from sqlbuild.compiler.planner._helpers.resolve.cursor_policies import ( - resolve_future_cursor_config, - resolve_start_cursor_config, -) -from sqlbuild.compiler.planner._helpers.resolve.refs import ( - apply_deferred_locations, - build_function_locations, - build_model_locations, - build_seed_locations, -) -from sqlbuild.compiler.planner._helpers.resolve.resolve import ( - resolve_model_hook_entries, - resolve_model_sql, -) -from sqlbuild.compiler.planner.constants import MICROBATCH_CONCURRENCY_MESSAGE -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety -from sqlbuild.compiler.planner.main.execution.future_cursor_warning import future_cursor_cap_warning -from sqlbuild.compiler.planner.main.execution.maximum_start_warning import maximum_start_cap_warning -from sqlbuild.compiler.planner.models import ( - BackfillResult, - ChangeDetectionResult, - CursorBounds, - CursorInputEvidence, - CursorInputRelation, - CursorOverridePair, - CursorOverrides, - DeferralInputs, - Duration, - EmptyCursorInputDecision, - MaximumStartPolicyInputs, - ModelCursorSnapshot, - ModelPlanContext, - ModelPlanEntry, - PlanEntryBuildInputs, - PlannerModelEntryResults, - PlannerRelationsContext, - PlannerResolvedActions, - PlannerScope, - PlanWarning, - ResolvedModelAction, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - CursorGrain, - CursorType, - IncrementalMode, - MaterializationType, - MicrobatchStrategy, - PlanAction, - PlanReason, - WarningSeverity, -) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.cursor_algebra.models import TimestampValue -from sqlbuild.cursor_algebra.types import BoundSentinel -from sqlbuild.spec.contracts.models import ( - FutureCursorsConfig, - LocalConfig, - ProjectConfig, - SourceEntry, - StartCursorsConfig, -) - - -@dataclass(frozen=True) -class _MicrobatchRangeInputs: - snapshot: WarehouseSnapshot - backfill: BackfillResult - start_cursor_override: str | None - end_cursor_override: str | None - runtime_owned_cursor_bounds: bool - cursor_input_relations: tuple[CursorInputRelation, ...] - future_cursor_config: FutureCursorsConfig | None - start_cursor_config: StartCursorsConfig | None - invocation_time: datetime | None - full_refresh: bool - replaces_relation: bool = False - - -def build_planner_relations_context( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - scope: PlannerScope, - deferral: DeferralInputs | None = None, - project_config: ProjectConfig | None = None, - local_config: LocalConfig | None = None, - known_source_columns: dict[str, tuple[ColumnInfo, ...]] | None = None, - relation_keys: frozenset[CompiledObjectKey] | None = None, -) -> PlannerRelationsContext: - """Resolve relation locations and source metadata for plan entry construction.""" - - effective_deferral: DeferralInputs = deferral if deferral is not None else DeferralInputs() - model_locations: dict[str, CompiledRelationLocation] = build_model_locations(project.models) - seed_locations: dict[str, CompiledRelationLocation] = build_seed_locations(project.seeds) - function_locations: dict[str, CompiledRelationLocation] = build_function_locations( - project.functions - ) - if effective_deferral.deferred_locations is not None: - model_locations, seed_locations = apply_deferred_locations( - model_locations=model_locations, - seed_locations=seed_locations, - deferred_locations=effective_deferral.deferred_locations, - selected_keys=scope.selected_keys, - ) - audit_relation_keys: frozenset[CompiledObjectKey] = _selected_audit_relation_keys( - project=project, - selected_keys=scope.selected_keys, - ) - source_map: dict[str, SourceEntry] = build_source_load_map( - project=project, - selected_keys=( - relation_keys - if relation_keys is not None - else scope.selected_keys | audit_relation_keys - ), - upstream_deps=scope.upstream_deps if scope.all_keys else None, - ) - source_read_map: dict[str, SourceEntry] = ( - build_source_read_map( - project=project, - source_map=source_map, - selected_keys=scope.selected_keys, - project_config=project_config, - local_config=local_config, - defer_sources_to=effective_deferral.defer_sources_to, - ) - if effective_deferral.source_deferral_enabled - else source_map - ) - python_source_map: dict[str, SourceEntry] = with_declared_source_reads( - project=project, source_map=source_map, source_names=scope.python_read_source_names - ) - python_source_read_map: dict[str, SourceEntry] = ( - build_source_read_map( - project=project, - source_map=python_source_map, - selected_keys=scope.selected_keys, - project_config=project_config, - local_config=local_config, - defer_sources_to=effective_deferral.defer_sources_to, - read_source_names=frozenset(python_source_map), - ) - if effective_deferral.source_deferral_enabled - else python_source_map - ) - source_columns: dict[str, tuple[ColumnInfo, ...]] - listed_source_names: frozenset[str] - source_columns, listed_source_names = _resolve_source_warehouse_columns( - project=project, - adapter=adapter, - connection=connection, - source_read_map=source_read_map, - known_source_columns=known_source_columns, - ) - return PlannerRelationsContext( - model_locations=model_locations, - seed_locations=seed_locations, - function_locations=function_locations, - source_map=source_map, - source_read_map=source_read_map, - python_source_read_map=python_source_read_map, - source_warehouse_columns=get_semantic_source_columns( - project=project, - adapter=adapter, - connection=connection, - source_read_map=source_read_map, - selected_keys=scope.selected_keys, - columns=source_columns, - ), - star_exclude_keyword=adapter.star_exclude_keyword(), - listed_source_names=listed_source_names, - ) - - -def _resolve_source_warehouse_columns( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - source_read_map: dict[str, SourceEntry], - known_source_columns: dict[str, tuple[ColumnInfo, ...]] | None, -) -> tuple[dict[str, tuple[ColumnInfo, ...]], frozenset[str]]: - """Reuse already-gathered source columns for the read map or gather them once.""" - - if known_source_columns is not None: - known: dict[str, tuple[ColumnInfo, ...]] = { - name: known_source_columns[name] - for name in source_read_map - if name in known_source_columns - } - return known, frozenset(known) - return gather_source_inspection( - project=project, - adapter=adapter, - connection=connection, - source_entries=tuple(source_read_map.values()), - ) - - -def plan_model( - *, - model: CompiledModelView, - snapshot: WarehouseSnapshot, - adapter: BaseAdapter, - context: ModelPlanContext, - sql_analysis_enabled: bool, - query_change_tracking: bool, - full_refresh: bool, - cursor_overrides: CursorOverridePair, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, -) -> tuple[ModelPlanEntry, tuple[PlanWarning, ...]]: - """Detect changes and build a plan entry and warnings for a single model.""" - - change_result: ChangeDetectionResult = detect_model_changes( - model=model, - snapshot=snapshot, - sql_analysis_enabled=sql_analysis_enabled, - query_change_tracking=query_change_tracking, - full_refresh=full_refresh, - ) - - return plan_model_from_change( - model=model, - snapshot=snapshot, - adapter=adapter, - context=context, - sql_analysis_enabled=sql_analysis_enabled, - full_refresh=full_refresh, - cursor_overrides=cursor_overrides, - change_result=change_result, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - - -def build_plan_entries( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - relations: PlannerRelationsContext, - resolved_actions: PlannerResolvedActions, - cursor_overrides: CursorOverrides | None, - full_refresh: bool, - build_inputs: PlanEntryBuildInputs | None = None, -) -> PlannerModelEntryResults: - """Build model plan entries from snapshot and per-model resolved actions.""" - - inputs: PlanEntryBuildInputs = ( - build_inputs if build_inputs is not None else PlanEntryBuildInputs() - ) - binding_diagnostics: tuple[CompilerDiagnostic, ...] = ( - *( - diagnostic - for diagnostic in project.diagnostics - if diagnostic.is_error and diagnostic.resource_type != CompiledResourceType.SQL_TEST - ), - *source_binding_diagnostics( - project=project, - columns=relations.source_warehouse_columns, - profile=adapter.expression_inference_profile(), - selected_keys=scope.selected_keys, - ), - ) - binding_diagnostics = tuple( - diagnostic for diagnostic in binding_diagnostics if diagnostic.is_error - ) - if binding_diagnostics: - raise PlannerInputError( - "\n".join( - f"error[{diagnostic.code}]: {diagnostic.message}\n" - f" --> {diagnostic.path}:{diagnostic.line or 1}:{diagnostic.column or 1}" - for diagnostic in binding_diagnostics - ), - code=binding_diagnostics[0].code, - ) - source_freshness_blocked_model_names: frozenset[str] = ( - inputs.source_freshness_blocked_model_names - ) - external_blocked_model_names: frozenset[str] = inputs.external_blocked_model_names - start_cursor_override: str | None = inputs.start_cursor_override - end_cursor_override: str | None = inputs.end_cursor_override - entries: list[ModelPlanEntry] = [] - warnings: list[PlanWarning] = [] - runtime_producer_names: frozenset[str] = runtime_cursor_producer_names( - scope=scope, - source_map=relations.source_map, - ) - key: CompiledObjectKey - for key in scope.execution_order: - if key not in scope.selected_keys or key.name not in resolved_actions.models: - continue - model: CompiledModelView | None = scope.models_by_name.get(key.name) - resolved: ResolvedModelAction = resolved_actions.models[key.name] - if model is None: - continue - resolved_start: str | None - resolved_end: str | None - resolved_start, resolved_end = resolve_cursor_overrides( - model=model, - cursor_overrides=cursor_overrides, - start_cursor_override=start_cursor_override, - end_cursor_override=end_cursor_override, - ) - entry: ModelPlanEntry - entry_warnings: tuple[PlanWarning, ...] - effective_full_refresh: bool = resolve_model_full_refresh( - model=model, - cli_full_refresh=full_refresh, - ) - entry, entry_warnings = plan_model_from_change( - model=model, - snapshot=snapshot, - adapter=adapter, - context=ModelPlanContext( - model_locations=relations.model_locations, - models_by_name=scope.models_by_name, - functions_by_name={function.name: function for function in project.functions}, - seed_locations=relations.seed_locations, - function_locations=relations.function_locations, - source_map=relations.source_read_map, - source_warehouse_columns=relations.source_warehouse_columns, - star_exclude_keyword=relations.star_exclude_keyword, - runtime_cursor_producer_names=runtime_producer_names, - future_cursor_config=inputs.future_cursor_config, - start_cursor_config=inputs.start_cursor_config, - invocation_time=inputs.invocation_time, - ), - sql_analysis_enabled=project.settings.sql_analysis, - full_refresh=effective_full_refresh, - cursor_overrides=CursorOverridePair( - start_cursor_override=resolved_start, - end_cursor_override=resolved_end, - ), - change_result=resolved.change, - external_sql_reference_resolver=project.external_sql_reference_resolver, - protect_full_refresh=key.name not in source_freshness_blocked_model_names - and key.name not in external_blocked_model_names, - ) - if entry.name in source_freshness_blocked_model_names: - entry = replace( - entry, - action=PlanAction.SKIP, - reason=PlanReason.SOURCE_FRESHNESS_ERROR, - ) - if entry.name in external_blocked_model_names: - entry = replace( - entry, - action=PlanAction.SKIP, - reason=PlanReason.EXTERNAL_UPSTREAM_FAILED, - ) - if entry.incremental_mode == IncrementalMode.MICROBATCH: - entry = replace( - entry, - unaccounted_partition_policy=( - entry.unaccounted_partition_policy - or project.settings.microbatch_unaccounted_partition_policy - ), - ) - if entry.action != PlanAction.SKIP: - entry, limit_warnings = apply_configured_microbatch_limits( - entry=entry, inputs=inputs - ) - warnings.extend(limit_warnings) - entries.append(entry) - warnings.extend(entry_warnings) - configured_concurrent_entries: tuple[ModelPlanEntry, ...] = tuple( - entry - for entry in entries - if entry.batch_concurrency > 1 and entry.action != PlanAction.SKIP - ) - if configured_concurrent_entries and not project.settings.microbatch_concurrency: - names: str = ", ".join(entry.name for entry in configured_concurrent_entries) - note, guidance = microbatch_guidance() - raise CompileInputError(f"{MICROBATCH_CONCURRENCY_MESSAGE}{names}; {note}", help=guidance) - if configured_concurrent_entries and not adapter.supports_concurrent_microbatch_dml(): - raise CompileInputError( - f"adapter '{adapter.adapter_name}' does not support concurrent microbatch " - "delete/insert DML; set batch_concurrency = 1" - ) - concurrent_entries: tuple[ModelPlanEntry, ...] = tuple( - entry for entry in configured_concurrent_entries if entry.action != PlanAction.CREATE_TABLE - ) - if concurrent_entries and project.settings.concurrency > 1: - count: int = len(concurrent_entries) - names = ", ".join(entry.name for entry in concurrent_entries) - warnings.append( - PlanWarning( - model_name=None, - severity=WarningSeverity.WARNING, - code="MICROBATCH_CONCURRENCY_COMPLEXITY", - message=( - f"Concurrent microbatch execution is enabled for {count} selected " - f"model{'s' if count != 1 else ''}: {names}. Concurrent batches may complete " - "out of order and rely on persistent microbatch history for gap " - "recovery and fingerprint provenance. State deletion or corruption " - "can require synthetic reconciliation and weaken version guarantees. " - "Keep batch_concurrency at 1 unless the runtime benefit justifies " - "this complexity; see the microbatch concurrency and recovery " - "documentation." - ), - ) - ) - return PlannerModelEntryResults(entries=tuple(entries), warnings=tuple(warnings)) - - -def plan_model_from_change( - *, - model: CompiledModelView, - snapshot: WarehouseSnapshot, - adapter: BaseAdapter, - context: ModelPlanContext, - sql_analysis_enabled: bool, - full_refresh: bool, - cursor_overrides: CursorOverridePair, - change_result: ChangeDetectionResult, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, - protect_full_refresh: bool = True, -) -> tuple[ModelPlanEntry, tuple[PlanWarning, ...]]: - """Build a model plan entry from a resolved change result.""" - - start_cursor_override: str | None = cursor_overrides.start_cursor_override - end_cursor_override: str | None = cursor_overrides.end_cursor_override - backfill: BackfillResult = ( - BackfillResult(action=BackfillAction.FULL) if full_refresh else change_result.backfill - ) - cursor_column: str | None = get_config_str(values=model.config.values, key="cursor") - validate_source_cursor_input_columns( - model=model, - cursor_column=cursor_column, - models_by_name=context.models_by_name, - functions_by_name=context.functions_by_name, - source_map=context.source_map, - source_warehouse_columns=context.source_warehouse_columns, - ) - - action: PlanAction - reason: PlanReason - action, reason = resolve_model_plan_action( - model=model, - change_result=change_result, - full_refresh=full_refresh, - ) - materialization_type: MaterializationType = get_materialization_type(model) - replaces_relation: bool = replaces_incremental_relation( - materialization_type=materialization_type, action=action - ) - full_rebuild_cause: str | None = ( - describe_full_rebuild_cause(change_result=change_result, full_refresh=full_refresh) - if replaces_relation - else None - ) - if full_rebuild_cause is not None and protect_full_refresh and not full_refresh: - check_full_rebuild_allowed( - model=model, change_result=change_result, full_rebuild_cause=full_rebuild_cause - ) - - resolved_sql: str = resolve_model_sql( - adapter=adapter, - model=model, - snapshot=snapshot, - context=context, - backfill=backfill, - full_refresh=full_refresh, - cursor_overrides=cursor_overrides, - external_sql_reference_resolver=external_sql_reference_resolver, - replaces_relation=replaces_relation, - full_rebuild_cause=full_rebuild_cause, - ) - - pre_hooks, post_hooks = resolve_model_hook_entries( - model=model, - adapter=adapter, - context=context, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - effective_future_cursor_config, effective_start_cursor_config = _resolve_cursor_safety_configs( - model=model, context=context - ) - validate_capped_watermark_inputs( - model=model, - models_by_name=context.models_by_name, - functions_by_name=context.functions_by_name, - ) - cursor_input_relations: tuple[CursorInputRelation, ...] = build_cursor_input_relations( - model=model, - adapter=adapter, - model_locations=context.model_locations, - models_by_name=context.models_by_name, - functions_by_name=context.functions_by_name, - seed_locations=context.seed_locations, - source_map=context.source_map, - cursor_column=cursor_column, - runtime_cursor_producer_names=context.runtime_cursor_producer_names, - ) - runtime_owned_cursor_bounds: bool = any( - relation.is_runtime_owned for relation in cursor_input_relations - ) and not (start_cursor_override is not None and end_cursor_override is not None) - cursor_bounds: CursorBounds | None = _compute_plan_cursor_bounds( - model=model, - snapshot=snapshot, - backfill=backfill, - full_refresh=full_refresh, - start_cursor_override=start_cursor_override, - end_cursor_override=end_cursor_override, - runtime_owned_cursor_bounds=runtime_owned_cursor_bounds, - future_cursor_config=effective_future_cursor_config, - start_cursor_config=effective_start_cursor_config, - invocation_time=context.invocation_time, - ) - - empty_cursor_inputs: tuple[str, ...] - waiting_on_empty_inputs: bool - empty_cursor_inputs, waiting_on_empty_inputs = _resolve_empty_cursor_inputs( - model=model, - snapshot=snapshot, - full_refresh=full_refresh, - runtime_owned_cursor_bounds=runtime_owned_cursor_bounds, - cursor_overrides=cursor_overrides, - ) - - microbatch_range: CursorBounds | None = _compute_microbatch_range( - model=model, - inputs=_MicrobatchRangeInputs( - snapshot=snapshot, - backfill=backfill, - start_cursor_override=start_cursor_override, - end_cursor_override=end_cursor_override, - runtime_owned_cursor_bounds=runtime_owned_cursor_bounds, - cursor_input_relations=cursor_input_relations, - future_cursor_config=effective_future_cursor_config, - start_cursor_config=effective_start_cursor_config, - invocation_time=context.invocation_time, - full_refresh=full_refresh, - replaces_relation=replaces_relation, - ), - ) - - entry: ModelPlanEntry - warnings: tuple[PlanWarning, ...] - entry, warnings = assemble_model_plan_entry( - model=model, - adapter=adapter, - snapshot=snapshot, - change_result=change_result, - action=action, - reason=reason, - materialization_type=materialization_type, - resolved_sql=resolved_sql, - sql_analysis_enabled=sql_analysis_enabled, - pre_hooks=pre_hooks, - post_hooks=post_hooks, - cursor_column=cursor_column, - cursor_input_relations=cursor_input_relations, - runtime_owned_cursor_bounds=runtime_owned_cursor_bounds, - cursor_bounds=cursor_bounds, - microbatch_range=microbatch_range, - empty_cursor_inputs=empty_cursor_inputs, - waiting_on_empty_inputs=waiting_on_empty_inputs, - backfill=backfill, - start_cursor_override=start_cursor_override, - end_cursor_override=end_cursor_override, - future_cursor_config=effective_future_cursor_config, - start_cursor_config=effective_start_cursor_config, - invocation_time=context.invocation_time, - ) - return entry, _append_cursor_safety_warnings( - warnings=warnings, - model_name=model.name, - bounds=microbatch_range or cursor_bounds, - ) - - -def resolve_cursor_overrides( - *, - model: CompiledModelView, - cursor_overrides: CursorOverrides | None, - start_cursor_override: str | None, - end_cursor_override: str | None, -) -> tuple[str | None, str | None]: - """Resolve typed cursor overrides to generic start/end for one model.""" - - cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") - resolved_start: str | None = start_cursor_override - resolved_end: str | None = end_cursor_override - - if cursor_overrides is not None and cursor_type == CursorType.TIMESTAMP: - if cursor_overrides.start_ts is not None: - resolved_start = render(value=cursor_overrides.start_ts) - if cursor_overrides.end_ts is not None: - resolved_end = render(value=cursor_overrides.end_ts) - elif cursor_overrides is not None and cursor_type == CursorType.INTEGER: - if cursor_overrides.start_int is not None: - resolved_start = render(value=cursor_overrides.start_int) - if cursor_overrides.end_int is not None: - resolved_end = render(value=cursor_overrides.end_int) - - cursor_start: str | None = get_config_cursor_bound( - values=model.config.values, key="cursor_start" - ) - cursor_end: str | None = get_config_cursor_bound(values=model.config.values, key="cursor_end") - if ( - resolved_start is not None - and cursor_start is not None - and _cursor_order_key(value=resolved_start, cursor_type=cursor_type) - < _cursor_order_key(value=cursor_start, cursor_type=cursor_type) - ): - raise CompileInputError( - f"model '{model.name}': explicit start {resolved_start} is before cursor_start " - f"{cursor_start}" - ) - if cursor_end is not None: - if resolved_start is not None and _cursor_order_key( - value=resolved_start, cursor_type=cursor_type - ) >= _cursor_order_key(value=cursor_end, cursor_type=cursor_type): - raise CompileInputError( - f"model '{model.name}': explicit start {resolved_start} is at or beyond " - f"exclusive cursor_end {cursor_end}" - ) - if resolved_end is not None and _cursor_order_key( - value=resolved_end, cursor_type=cursor_type - ) >= _cursor_order_key(value=cursor_end, cursor_type=cursor_type): - raise CompileInputError( - f"model '{model.name}': explicit end {resolved_end} is at or beyond " - f"exclusive cursor_end {cursor_end}" - ) - return resolved_start, resolved_end - - -def _resolve_cursor_safety_configs( - *, model: CompiledModelView, context: ModelPlanContext -) -> tuple[FutureCursorsConfig | None, StartCursorsConfig | None]: - return ( - resolve_future_cursor_config(model=model, project_config=context.future_cursor_config), - resolve_start_cursor_config(model=model, project_config=context.start_cursor_config), - ) - - -def _compute_microbatch_range( - *, model: CompiledModelView, inputs: _MicrobatchRangeInputs -) -> CursorBounds | None: - """Compute the real overall cursor range for microbatch batch splitting.""" - - materialized: str | None = get_config_str(values=model.config.values, key="materialized") - if materialized != MaterializationType.INCREMENTAL: - return None - incremental_mode: str | None = get_config_str( - values=model.config.values, key="incremental_mode" - ) - if incremental_mode != IncrementalMode.MICROBATCH: - return None - cursor_column: str | None = get_config_str(values=model.config.values, key="cursor") - if cursor_column is None: - return None - bounded_override: CursorBounds | None = resolve_bounded_cursor_override( - start_cursor_override=inputs.start_cursor_override, - end_cursor_override=inputs.end_cursor_override, - cursor_type=get_config_str(values=model.config.values, key="cursor_type"), - cursor_grain=get_config_str(values=model.config.values, key="cursor_grain"), - ) - if bounded_override is not None: - return _clamp_cursor_end(bounds=bounded_override, model=model) - if ( - get_config_str(values=model.config.values, key="microbatch_strategy") - == MicrobatchStrategy.ROLLING_WINDOW - ): - rolling_bounds: CursorBounds | None = _rolling_window_bounds( - model=model, invocation_time=inputs.invocation_time - ) - if inputs.full_refresh and rolling_bounds is not None: - cursor_start: str | None = get_config_cursor_bound( - values=model.config.values, key="cursor_start" - ) - if cursor_start is not None: - rolling_bounds = replace(rolling_bounds, start=cursor_start) - return rolling_bounds - if inputs.runtime_owned_cursor_bounds: - return None - cursor_snapshot: ModelCursorSnapshot | None = inputs.snapshot.cursor_snapshots.get(model.name) - if cursor_snapshot is None: - return None - if inputs.replaces_relation and not inputs.full_refresh: - cursor_snapshot = without_destination_cursor(cursor_snapshot=cursor_snapshot) - - cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") - downstream_grain: str | None = get_config_str(values=model.config.values, key="cursor_grain") - effective_grain: str | None = resolve_effective_timestamp_grain( - cursor_type=cursor_type, - downstream_grain=downstream_grain, - microbatch_strategy=get_config_str(values=model.config.values, key="microbatch_strategy"), - cursor_input_grains=tuple( - relation.cursor_grain for relation in inputs.cursor_input_relations - ), - ) - if cursor_type == CursorType.TIMESTAMP and effective_grain is not None: - cursor_snapshot = normalize_cursor_snapshot_grain( - cursor_snapshot=cursor_snapshot, - cursor_type=cursor_type, - effective_grain=effective_grain, - ) - - lookback: str | None = resolve_microbatch_lookback(model=model, effective_grain=effective_grain) - cursor_start: str | None = get_config_cursor_bound( - values=model.config.values, key="cursor_start" - ) - backfill_duration: str | None = None - if inputs.backfill.action == BackfillAction.BOUNDED: - backfill_duration = inputs.backfill.duration - - bounds: CursorBounds | None = compute_cursor_bounds( - cursor_snapshot=cursor_snapshot, - cursor_type=cursor_type, - cursor_start=cursor_start, - lookback=lookback, - backfill_duration=backfill_duration, - start_cursor_override=inputs.start_cursor_override, - end_cursor_override=inputs.end_cursor_override, - cursor_grain=effective_grain, - is_microbatch=False, - maximum_start_policy=MaximumStartPolicyInputs( - config=inputs.start_cursor_config, - invocation_time=inputs.invocation_time, - incremental_strategy=get_config_str( - values=model.config.values, key="incremental_strategy" - ), - incremental_mode=incremental_mode, - ), - ) - return _clamp_cursor_end( - bounds=_apply_future_safety( - bounds=bounds, - cursor_type=cursor_type, - cursor_grain=effective_grain, - start_cursor_override=inputs.start_cursor_override, - end_cursor_override=inputs.end_cursor_override, - future_cursor_config=inputs.future_cursor_config, - invocation_time=inputs.invocation_time, - input_evidence=cursor_snapshot.input_evidence, - ), - model=model, - ) - - -def _compute_plan_cursor_bounds( - *, - model: CompiledModelView, - snapshot: WarehouseSnapshot, - backfill: BackfillResult, - full_refresh: bool, - start_cursor_override: str | None, - end_cursor_override: str | None, - runtime_owned_cursor_bounds: bool, - future_cursor_config: FutureCursorsConfig | None = None, - start_cursor_config: StartCursorsConfig | None = None, - invocation_time: datetime | None = None, -) -> CursorBounds | None: - """Compute cursor bounds for inclusion on the plan entry.""" - - materialized: str | None = get_config_str(values=model.config.values, key="materialized") - cursor_column: str | None = get_config_str(values=model.config.values, key="cursor") - if materialized != MaterializationType.INCREMENTAL or cursor_column is None: - return None - bounded_override: CursorBounds | None = resolve_bounded_cursor_override( - start_cursor_override=start_cursor_override, - end_cursor_override=end_cursor_override, - cursor_type=get_config_str(values=model.config.values, key="cursor_type"), - cursor_grain=get_config_str(values=model.config.values, key="cursor_grain"), - ) - if bounded_override is not None: - return _clamp_cursor_end(bounds=bounded_override, model=model) - if runtime_owned_cursor_bounds: - return None - if full_refresh: - return None - - cursor_snapshot: ModelCursorSnapshot | None = snapshot.cursor_snapshots.get(model.name) - if cursor_snapshot is None: - return None - if backfill.action == BackfillAction.FULL: - cursor_snapshot = without_destination_cursor(cursor_snapshot=cursor_snapshot) - - lookback: str | None = get_config_str(values=model.config.values, key="lookback") - cursor_start: str | None = get_config_cursor_bound( - values=model.config.values, key="cursor_start" - ) - incremental_mode: str | None = get_config_str( - values=model.config.values, key="incremental_mode" - ) - is_microbatch: bool = incremental_mode == IncrementalMode.MICROBATCH - - backfill_duration: str | None = None - if backfill.action == BackfillAction.BOUNDED: - backfill_duration = backfill.duration - - cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") - cursor_grain: str | None = get_config_str(values=model.config.values, key="cursor_grain") - bounds: CursorBounds | None = compute_cursor_bounds( - cursor_snapshot=cursor_snapshot, - cursor_type=cursor_type, - cursor_start=cursor_start, - lookback=lookback, - backfill_duration=backfill_duration, - start_cursor_override=start_cursor_override, - end_cursor_override=end_cursor_override, - is_microbatch=is_microbatch, - cursor_grain=cursor_grain, - maximum_start_policy=MaximumStartPolicyInputs( - config=start_cursor_config, - invocation_time=invocation_time, - incremental_strategy=get_config_str( - values=model.config.values, key="incremental_strategy" - ), - incremental_mode=incremental_mode, - ), - ) - return _clamp_cursor_end( - bounds=_apply_future_safety( - bounds=bounds, - cursor_type=cursor_type, - cursor_grain=cursor_grain, - start_cursor_override=start_cursor_override, - end_cursor_override=end_cursor_override, - future_cursor_config=future_cursor_config, - invocation_time=invocation_time, - input_evidence=cursor_snapshot.input_evidence, - ), - model=model, - ) - - -def _resolve_empty_cursor_inputs( - *, - model: CompiledModelView, - snapshot: WarehouseSnapshot, - full_refresh: bool, - runtime_owned_cursor_bounds: bool, - cursor_overrides: CursorOverridePair, -) -> tuple[tuple[str, ...], bool]: - """Return empty inputs that leave a planner-owned window empty, and whether others have rows.""" - - if ( - full_refresh - or runtime_owned_cursor_bounds - or get_config_str(values=model.config.values, key="materialized") - != MaterializationType.INCREMENTAL - or get_config_str(values=model.config.values, key="cursor") is None - or get_config_str(values=model.config.values, key="microbatch_strategy") - == MicrobatchStrategy.ROLLING_WINDOW - or ( - cursor_overrides.start_cursor_override is not None - and cursor_overrides.end_cursor_override is not None - ) - ): - return (), False - cursor_snapshot: ModelCursorSnapshot | None = snapshot.cursor_snapshots.get(model.name) - if cursor_snapshot is None: - return (), False - decision: EmptyCursorInputDecision = cursor_inputs_without_rows(cursor_snapshot=cursor_snapshot) - return decision.input_names, decision.waiting_on_empty_inputs - - -def _apply_future_safety( - *, - bounds: CursorBounds | None, - cursor_type: str | None, - cursor_grain: str | None, - start_cursor_override: str | None, - end_cursor_override: str | None, - future_cursor_config: FutureCursorsConfig | None, - invocation_time: datetime | None, - input_evidence: tuple[CursorInputEvidence, ...], -) -> CursorBounds | None: - if bounds is None: - return None - if bounds.end == BoundSentinel.END: - return bounds - return apply_future_cursor_safety( - bounds=bounds, - cursor_type=cursor_type, - cursor_grain=cursor_grain, - config=future_cursor_config, - invocation_time=invocation_time, - has_complete_override=( - start_cursor_override is not None and end_cursor_override is not None - ), - input_evidence=input_evidence, - ) - - -def _append_future_cap_warning( - *, warnings: tuple[PlanWarning, ...], model_name: str, bounds: CursorBounds | None -) -> tuple[PlanWarning, ...]: - message: str | None = future_cursor_cap_warning(bounds) - if message is None: - return warnings - return ( - *warnings, - PlanWarning( - model_name=model_name, - severity=WarningSeverity.WARNING, - code="FUTURE_CURSOR_CAPPED", - message=message, - ), - ) - - -def _append_cursor_safety_warnings( - *, warnings: tuple[PlanWarning, ...], model_name: str, bounds: CursorBounds | None -) -> tuple[PlanWarning, ...]: - with_future: tuple[PlanWarning, ...] = _append_future_cap_warning( - warnings=warnings, model_name=model_name, bounds=bounds - ) - return _append_maximum_start_warning(warnings=with_future, model_name=model_name, bounds=bounds) - - -def _append_maximum_start_warning( - *, warnings: tuple[PlanWarning, ...], model_name: str, bounds: CursorBounds | None -) -> tuple[PlanWarning, ...]: - message: str | None = maximum_start_cap_warning(bounds) - if message is None: - return warnings - return ( - *warnings, - PlanWarning( - model_name=model_name, - severity=WarningSeverity.WARNING, - code="AUTOMATIC_START_CAPPED", - message=message, - ), - ) - - -def scope_overlaps( - *, - scope_deps: tuple[CompiledObjectKey, ...], - selected_keys: frozenset[CompiledObjectKey], -) -> bool: - """Check if any scope dependency is in the selected keys.""" - - dep: CompiledObjectKey - for dep in scope_deps: - if dep in selected_keys: - return True - return False - - -def _selected_audit_relation_keys( - *, - project: CompiledProjectView, - selected_keys: frozenset[CompiledObjectKey], -) -> frozenset[CompiledObjectKey]: - """Return relation keys needed to render selected audits without selecting those relations.""" - - relation_keys: set[CompiledObjectKey] = set() - audit: CompiledAuditView - for audit in project.audits: - if audit_is_selected(audit=audit, selected_keys=selected_keys): - relation_keys.update(audit.scope_deps) - return frozenset(relation_keys) - - -def build_model_materializations( - model_entries: tuple[ModelPlanEntry, ...], -) -> dict[str, str]: - """Build a name-to-materialization-type lookup from planned model entries.""" - - return {entry.name: entry.materialization_type for entry in model_entries} - - -def _rolling_window_bounds( - *, model: CompiledModelView, invocation_time: datetime | None -) -> CursorBounds | None: - grain: str | None = get_config_str(values=model.config.values, key="cursor_grain") - lookback: str | None = resolve_microbatch_lookback(model=model, effective_grain=grain) - duration: Duration | None = Duration.parse(lookback or "") - if grain is None or duration is None: - return None - moment: datetime = invocation_time or datetime.now(tz=UTC) - if moment.tzinfo is None: - moment = moment.replace(tzinfo=UTC) - else: - moment = moment.astimezone(UTC) - anchor: datetime = _floor_utc_grain(moment=moment, grain=grain) - bounds: CursorBounds = CursorBounds( - start=TimestampValue(value=duration.subtract_from(anchor)), - end=TimestampValue(value=anchor), - ) - cursor_start: str | None = get_config_cursor_bound( - values=model.config.values, key="cursor_start" - ) - if cursor_start is not None and _cursor_order_key( - value=sentinel_to_token(sentinel=bounds.start), cursor_type=CursorType.TIMESTAMP - ) < _cursor_order_key(value=cursor_start, cursor_type=CursorType.TIMESTAMP): - bounds = replace(bounds, start=cursor_start) - return _clamp_cursor_end(bounds=bounds, model=model) - - -def _floor_utc_grain(*, moment: datetime, grain: str) -> datetime: - if grain == CursorGrain.YEAR: - return moment.replace(month=1, day=1, hour=0, minute=0, second=0, microsecond=0) - if grain == CursorGrain.MONTH: - return moment.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - if grain == CursorGrain.DAY: - return moment.replace(hour=0, minute=0, second=0, microsecond=0) - if grain == CursorGrain.HOUR: - return moment.replace(minute=0, second=0, microsecond=0) - if grain == CursorGrain.MINUTE: - return moment.replace(second=0, microsecond=0) - return moment.replace(microsecond=0) - - -def _clamp_cursor_end( - *, bounds: CursorBounds | None, model: CompiledModelView -) -> CursorBounds | None: - cursor_end: str | None = get_config_cursor_bound(values=model.config.values, key="cursor_end") - cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") - if bounds is None or cursor_end is None: - return bounds - if bounds.start == BoundSentinel.START or bounds.end == BoundSentinel.END: - return bounds - if _cursor_order_key( - value=sentinel_to_token(sentinel=bounds.start), cursor_type=cursor_type - ) >= _cursor_order_key(value=cursor_end, cursor_type=cursor_type): - return replace(bounds, start=cursor_end, end=cursor_end) - if _cursor_order_key( - value=sentinel_to_token(sentinel=bounds.end), cursor_type=cursor_type - ) > _cursor_order_key(value=cursor_end, cursor_type=cursor_type): - return replace(bounds, end=cursor_end) - return bounds - - -def _cursor_order_key(*, value: str, cursor_type: str | None) -> Decimal: - if cursor_type == CursorType.INTEGER: - return Decimal(value) - parsed: datetime = datetime.fromisoformat(value) - if parsed.tzinfo is not None: - parsed = parsed.astimezone(UTC).replace(tzinfo=None) - return Decimal(str(parsed.replace(tzinfo=UTC).timestamp())) diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py deleted file mode 100644 index 850550e264..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py +++ /dev/null @@ -1,500 +0,0 @@ -"""PlanOutput assembly helpers.""" - -from __future__ import annotations - -from sqlbuild._native import ( - build_relation_fixture_context, - build_selected_audit_entries, - build_source_load_entries, - extract_seed_columns, - plan_sql_tests, - upstream_loader_dependency_names, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledSqlTest, -) -from sqlbuild.compiler.compile.types import ( - CompiledFunctionView, - CompiledProjectView, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredHookFunction, - DiscoveredLoaderFunction, - DiscoveredMaterializationFile, - DiscoveredProviderUsage, - PythonHookEntry, -) -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.output.plan_entry import ( - build_model_materializations, - scope_overlaps, -) -from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_function_sql -from sqlbuild.compiler.planner.exceptions import ( - PlannerInputError, -) -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - FunctionChangeResult, - FunctionPlanEntry, - ModelPlanEntry, - PlannedSqlTests, - PlannerChangeResults, - PlannerModelEntryResults, - PlannerRelationsContext, - PlannerScope, - PlanOutput, - PlanOutputExtras, - PlanProviderUsage, - PlanWarning, - RelationFixturePlanningContext, - SeedPlanEntry, - SourceLoadPlanEntry, - SqlTestPlanEntry, - SqlTestPlanResult, - SqlTestSelection, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import PlanReason -from sqlbuild.spec.contracts.models import SourceEntry - - -def build_plan_output( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - relations: PlannerRelationsContext, - changes: PlannerChangeResults, - model_entry_results: PlannerModelEntryResults, - reload_sources: bool, - extras: PlanOutputExtras | None = None, -) -> PlanOutput: - resolved_extras: PlanOutputExtras = extras if extras is not None else PlanOutputExtras() - seed_version_hashes: dict[str, str] | None = resolved_extras.seed_version_hashes - seed_metadata_jsons: dict[str, str] | None = resolved_extras.seed_metadata_jsons - seed_plan_reasons: dict[str, PlanReason] | None = resolved_extras.seed_plan_reasons - seed_entries: list[SeedPlanEntry] = [ - SeedPlanEntry( - key=seed.key, - name=seed.name, - destination=seed.destination, - file_path=seed.seed_file.file_path, - columns=extract_seed_columns(seed), - csv_settings=seed.schema_entry.csv_settings, - fingerprint_definition=(seed_metadata_jsons or {}).get(seed.name, ""), - fingerprint_version_hash=(seed_version_hashes or {}).get(seed.name, ""), - fingerprint_metadata_json=(seed_metadata_jsons or {}).get(seed.name, "{}"), - reason=_resolve_seed_plan_reason( - seed_name=seed.name, - expected_version_hash=(seed_version_hashes or {}).get(seed.name), - snapshot=snapshot, - seed_plan_reasons=seed_plan_reasons, - ), - ) - for seed in project.seeds - if seed.key in scope.selected_keys - ] - source_load_entries: tuple[SourceLoadPlanEntry, ...] = build_source_load_entries( - execution_order=scope.execution_order, - selected_keys=scope.selected_keys, - source_map=relations.source_map, - is_reload=reload_sources, - ) - _validate_skipped_intermediate_loader_targets( - project=project, - snapshot=snapshot, - relations=relations, - source_load_entries=source_load_entries, - ) - function_entries: list[FunctionPlanEntry] = _build_function_entries( - project=project, - adapter=adapter, - snapshot=snapshot, - relations=relations, - changes=changes, - selected_keys=scope.selected_keys, - ) - model_materializations: dict[str, str] = build_model_materializations( - model_entry_results.entries - ) - audit_entries: list[AuditPlanEntry] = build_selected_audit_entries( - project=project, - adapter=adapter, - scope=scope, - relations=relations, - model_materializations=model_materializations, - ) - planned_tests: PlannedSqlTests | None = resolved_extras.planned_sql_tests - if ( - planned_tests is None - or planned_tests.selected_keys != scope.selected_keys - or planned_tests.sql_test_selection != scope.sql_test_selection - ): - planned_tests = plan_selected_sql_tests( - project=project, - adapter=adapter, - selected_keys=scope.selected_keys, - sql_test_selection=scope.sql_test_selection, - ) - test_entries: list[SqlTestPlanEntry] = list(planned_tests.entries) - test_warnings: list[PlanWarning] = list(planned_tests.warnings) - selected_test_keys: frozenset[CompiledObjectKey] = frozenset( - entry.key for entry in test_entries - ) - scoped_keys: frozenset[CompiledObjectKey] = scope.selected_keys | selected_test_keys - scoped_order: tuple[CompiledObjectKey, ...] = tuple( - k for k in scope.execution_order if k in scoped_keys - ) - return PlanOutput( - execution_order=scoped_order, - model_entries=model_entry_results.entries, - seed_entries=tuple(seed_entries), - source_load_entries=tuple(source_load_entries), - function_entries=tuple(function_entries), - audit_entries=tuple(audit_entries), - test_entries=tuple(test_entries), - selected_keys=scope.selected_keys, - warnings=(*model_entry_results.warnings, *test_warnings), - upstream_deps=scope.upstream_deps, - downstream_deps=scope.downstream_deps, - model_locations=relations.model_locations, - seed_locations=relations.seed_locations, - function_locations=relations.function_locations, - source_map=relations.source_map, - source_read_map=relations.source_read_map, - python_source_read_map=relations.python_source_read_map, - hook_functions=project.hook_functions, - enforce_explicit_references=project.enforce_explicit_references, - provider_usages=_build_provider_usages( - project=project, - model_entries=model_entry_results.entries, - source_load_entries=source_load_entries, - ), - python_identity_fingerprints=snapshot.fingerprints.python_nodes, - ) - - -def _resolve_seed_plan_reason( - *, - seed_name: str, - expected_version_hash: str | None, - snapshot: WarehouseSnapshot, - seed_plan_reasons: dict[str, PlanReason] | None = None, -) -> PlanReason: - reason: PlanReason | None = (seed_plan_reasons or {}).get(seed_name) - if reason is not None: - return reason - if seed_name not in snapshot.existing_relations: - return PlanReason.FIRST_RUN - fingerprint: Fingerprint | None = snapshot.fingerprints.seeds.get(seed_name) - if fingerprint is None: - return PlanReason.FIRST_RUN - if expected_version_hash is not None and fingerprint.version_hash != expected_version_hash: - return PlanReason.CONFIG_CHANGED - return PlanReason.NO_CHANGE - - -def _build_provider_usages( - *, - project: CompiledProjectView, - model_entries: tuple[ModelPlanEntry, ...], - source_load_entries: tuple[SourceLoadPlanEntry, ...], -) -> tuple[PlanProviderUsage, ...]: - usages: list[PlanProviderUsage] = [] - loader_by_name: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in project.loader_functions - } - source_load_entry: SourceLoadPlanEntry - for source_load_entry in source_load_entries: - loader: DiscoveredLoaderFunction | None = loader_by_name.get(source_load_entry.loader) - if loader is None: - continue - usages.extend( - _to_plan_provider_usages( - provider_usages=loader.provider_usages, - consumer_kind="loader", - consumer_name=loader.name, - ) - ) - - hook_by_name: dict[str, DiscoveredHookFunction] = { - hook.name: hook for hook in project.hook_functions - } - materialization_by_name: dict[str, DiscoveredMaterializationFile] = { - materialization.name: materialization for materialization in project.materialization_files - } - model_entry: ModelPlanEntry - for model_entry in model_entries: - usages.extend( - _hook_provider_usages( - hook_entries=(model_entry.pre_hooks, model_entry.post_hooks), - hook_by_name=hook_by_name, - ) - ) - if model_entry.custom_materialization_name is None: - continue - materialization: DiscoveredMaterializationFile | None = materialization_by_name.get( - model_entry.custom_materialization_name - ) - if materialization is None: - continue - usages.extend( - _to_plan_provider_usages( - provider_usages=materialization.provider_usages, - consumer_kind="custom materialization", - consumer_name=materialization.name, - ) - ) - return tuple(usages) - - -def _hook_provider_usages( - *, - hook_entries: tuple[object, ...], - hook_by_name: dict[str, DiscoveredHookFunction], -) -> tuple[PlanProviderUsage, ...]: - usages: list[PlanProviderUsage] = [] - hooks: list[object] = [] - hook_entry: object - for hook_entry in hook_entries: - if hook_entry is None: - continue - if isinstance(hook_entry, list | tuple): - hooks.extend(hook_entry) - else: - hooks.append(hook_entry) - hook: object - for hook in hooks: - if not isinstance(hook, PythonHookEntry): - continue - discovered_hook: DiscoveredHookFunction | None = hook_by_name.get(hook.name) - if discovered_hook is None: - continue - usages.extend( - _to_plan_provider_usages( - provider_usages=discovered_hook.provider_usages, - consumer_kind="hook", - consumer_name=discovered_hook.name, - ) - ) - return tuple(usages) - - -def _to_plan_provider_usages( - *, - provider_usages: tuple[DiscoveredProviderUsage, ...], - consumer_kind: str, - consumer_name: str, -) -> tuple[PlanProviderUsage, ...]: - return tuple( - PlanProviderUsage( - provider_name=usage.provider_name, - consumer_kind=consumer_kind, - consumer_name=consumer_name, - parameter_name=usage.parameter_name, - annotation_class_name=usage.annotation_class_name, - annotation_module=usage.annotation_module, - ) - for usage in provider_usages - ) - - -def _validate_skipped_intermediate_loader_targets( - *, - project: CompiledProjectView, - snapshot: WarehouseSnapshot, - relations: PlannerRelationsContext, - source_load_entries: tuple[SourceLoadPlanEntry, ...], -) -> None: - loader_by_name: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in project.loader_functions - } - selected_load_names: frozenset[str] = frozenset(entry.name for entry in source_load_entries) - entry: SourceLoadPlanEntry - for entry in source_load_entries: - source_entry: SourceEntry | None = relations.source_map.get(entry.name) - if source_entry is None or source_entry.loader is None: - continue - loader_function: DiscoveredLoaderFunction | None = loader_by_name.get(source_entry.loader) - if loader_function is None: - continue - dependency_loader_name: str - for dependency_loader_name in upstream_loader_dependency_names( - loader_function=loader_function, - loader_functions=project.loader_functions, - ): - dependency_source: SourceEntry | None = relations.source_map.get(dependency_loader_name) - if dependency_source is None or dependency_source.name in selected_load_names: - continue - dependency_target: str = dependency_source.table or dependency_source.name - if dependency_target in snapshot.existing_relations: - continue - raise PlannerInputError( - f"Source '{source_entry.name}' requires intermediate loader " - f"'{dependency_loader_name}', but its target relation " - f"'{dependency_target}' does not exist; use +source:{source_entry.name} " - "to refresh upstream ingress dependencies" - ) - - -def _build_function_entries( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - snapshot: WarehouseSnapshot, - relations: PlannerRelationsContext, - changes: PlannerChangeResults, - selected_keys: frozenset[CompiledObjectKey], -) -> list[FunctionPlanEntry]: - entries: list[FunctionPlanEntry] = [] - function: CompiledFunctionView - for function in project.functions: - if function.key not in selected_keys: - continue - function_change: FunctionChangeResult = changes.functions[function.name] - entries.append( - plan_function( - function=function, - adapter=adapter, - relations=relations, - fingerprint_query_sql=function_change.fingerprint_sql, - previous_query_sql=( - snapshot.fingerprints.functions[function.name].definition - if function.name in snapshot.fingerprints.functions - else None - ), - reason=function_change.reason, - ) - ) - return entries - - -def plan_function( - *, - function: CompiledFunctionView, - adapter: BaseAdapter, - relations: PlannerRelationsContext, - fingerprint_query_sql: str, - previous_query_sql: str | None = None, - reason: PlanReason = PlanReason.NO_CHANGE, -) -> FunctionPlanEntry: - """Build one canonical function entry from resolved relation inputs.""" - - return FunctionPlanEntry( - key=function.key, - name=function.name, - relative_path=function.relative_path, - destination=function.destination, - arguments=function.arguments, - returns=function.returns, - body_sql=resolve_function_sql( - adapter=adapter, - function=function, - model_locations=relations.model_locations, - seed_locations=relations.seed_locations, - function_locations=relations.function_locations, - source_map=relations.source_read_map, - source_warehouse_columns=relations.source_warehouse_columns, - star_exclude_keyword=relations.star_exclude_keyword, - ), - fingerprint_query_sql=fingerprint_query_sql, - fingerprint_destination=function.fingerprint_destination, - return_columns=function.return_columns, - language=function.language, - source_file_path=function.source_file_path, - runtime_version=function.runtime_version, - entry_point=function.entry_point, - packages=function.packages, - previous_query_sql=previous_query_sql, - reason=reason, - ) - - -def plan_selected_sql_tests( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, -) -> PlannedSqlTests: - """Plan SQL tests selected by name or overlapping one selection into a reusable result.""" - - entries: list[SqlTestPlanEntry] - warnings: list[PlanWarning] - entries, warnings = build_selected_test_entries( - project=project, - adapter=adapter, - selected_keys=selected_keys, - sql_test_selection=sql_test_selection, - ) - return PlannedSqlTests( - selected_keys=selected_keys, - sql_test_selection=sql_test_selection, - entries=tuple(entries), - warnings=tuple(warnings), - ) - - -def sql_test_is_selected( - *, - test: CompiledSqlTest, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, -) -> bool: - """Whether a test is named by the selection or covers a selected resource, and not excluded.""" - - name: str = test.parent_name or test.name - if name in sql_test_selection.excluded_names: - return False - return name in sql_test_selection.names or scope_overlaps( - scope_deps=test.scope_deps, selected_keys=selected_keys - ) - - -def build_selected_test_entries( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, - case_name: str | None = None, -) -> tuple[list[SqlTestPlanEntry], list[PlanWarning]]: - entries: list[SqlTestPlanEntry] = [] - warnings: list[PlanWarning] = [] - fixture_diagnostics: list[str] = [] - selected_tests: list[CompiledSqlTest] = [] - sql_test: CompiledSqlTest - for sql_test in project.sql_tests: - if not sql_test_is_selected( - test=sql_test, selected_keys=selected_keys, sql_test_selection=sql_test_selection - ): - continue - if case_name is not None and sql_test.case_name != case_name: - continue - selected_tests.append(sql_test) - if not selected_tests: - return entries, warnings - fixture_planning_context: RelationFixturePlanningContext | None = ( - build_relation_fixture_context(project=project) if project.settings.sql_analysis else None - ) - result: SqlTestPlanResult - for result in plan_sql_tests( - tests=tuple(selected_tests), - project=project, - adapter=adapter, - sql_analysis_enabled=project.settings.sql_analysis, - validate_fixtures=True, - fixture_planning_context=fixture_planning_context, - ): - if result.entry is None: - fixture_diagnostics.extend(result.fixture_diagnostics) - continue - entries.append(result.entry) - warnings.extend(result.warnings) - if fixture_diagnostics: - raise PlannerInputError("Invalid SQL test fixtures:\n- " + "\n- ".join(fixture_diagnostics)) - return entries, warnings diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/__init__.py b/src/sqlbuild/compiler/planner/_helpers/resolve/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/config.py b/src/sqlbuild/compiler/planner/_helpers/resolve/config.py deleted file mode 100644 index 26698e7dcc..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/config.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Model config extraction helpers for resolve modules.""" - -from __future__ import annotations - -from sqlbuild._native import get_config_str -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner.types import IncrementalStrategy - - -def get_config_append_cursor_inclusive(model: CompiledModelView) -> bool: - """Return effective append cursor lower-bound inclusivity.""" - - strategy: str | None = get_config_str(values=model.config.values, key="incremental_strategy") - if strategy != IncrementalStrategy.APPEND: - return True - raw: object | None = model.config.values.get("append_cursor_inclusive") - if isinstance(raw, bool): - return raw - return True diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py deleted file mode 100644 index cfa1fa0d30..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Native cursor override typing and cursor-window exports.""" - -from sqlbuild._native import ( - apply_typed_cursor_replay_policy as apply_typed_cursor_replay_policy, -) -from sqlbuild._native import ( - compute_cursor_bounds as compute_cursor_bounds, -) -from sqlbuild._native import ( - cursor_inputs_without_rows as cursor_inputs_without_rows, -) -from sqlbuild._native import ( - cursor_window_override_flags as cursor_window_override_flags, -) -from sqlbuild._native import ( - decide_empty_cursor_inputs as decide_empty_cursor_inputs, -) -from sqlbuild._native import ( - empty_cursor_window as empty_cursor_window, -) -from sqlbuild._native import ( - empty_input_rebuild_refusal as empty_input_rebuild_refusal, -) -from sqlbuild._native import ( - normalize_cursor_snapshot_grain as normalize_cursor_snapshot_grain, -) -from sqlbuild._native import ( - resolve_effective_timestamp_grain as resolve_effective_timestamp_grain, -) -from sqlbuild._native import ( - type_cursor_overrides as type_cursor_overrides, -) -from sqlbuild._native import ( - unreadable_cursor_inputs_error as unreadable_cursor_inputs_error, -) -from sqlbuild._native import ( - without_destination_cursor as without_destination_cursor, -) diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py deleted file mode 100644 index b819708b61..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Render cursor intrinsics with one interval's adapter-typed bound literals.""" - -from __future__ import annotations - -from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild._native import resolve_cursor_intrinsics -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import CursorBounds - - -def render_cursor_intrinsic_bounds( - *, sql: str, bounds: CursorBounds, cursor_type: str | None, adapter: BaseAdapter -) -> str: - """Replace `__cursor_start()` / `__cursor_end()` with the adapter's typed bound literals.""" - - rendered, _ = resolve_cursor_intrinsics( - sql=sql, - start_sql=adapter.render_cursor_bound_literal( - value=sentinel_to_token(sentinel=bounds.start), cursor_type=cursor_type - ), - end_sql=adapter.render_cursor_bound_literal( - value=sentinel_to_token(sentinel=bounds.end), cursor_type=cursor_type - ), - ) - return rendered diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_policies.py b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_policies.py deleted file mode 100644 index f1885743da..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_policies.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Resolve model-level cursor policy overrides.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.spec.contracts.constants import CURSOR_POLICY_DISABLED -from sqlbuild.spec.contracts.models import FutureCursorsConfig, StartCursorsConfig -from sqlbuild.spec.contracts.types import FutureCursorAction - - -def resolve_start_cursor_config( - *, model: CompiledModelView, project_config: StartCursorsConfig | None -) -> StartCursorsConfig | None: - """Resolve automatic-start policy with model config taking precedence.""" - - configured: StartCursorsConfig = project_config or StartCursorsConfig() - raw_distance: object | None = model.config.values.get("cursor_start_max_ahead") - raw_action: object | None = model.config.values.get("cursor_start_max_action") - max_ahead: str | None = configured.max_ahead - if isinstance(raw_distance, str): - max_ahead = None if raw_distance == CURSOR_POLICY_DISABLED else raw_distance - action: FutureCursorAction = configured.action - if isinstance(raw_action, str): - action = FutureCursorAction(raw_action) - return StartCursorsConfig(max_ahead=max_ahead, action=action) if max_ahead is not None else None - - -def resolve_future_cursor_config( - *, model: CompiledModelView, project_config: FutureCursorsConfig | None -) -> FutureCursorsConfig | None: - """Resolve future-end policy with model config taking precedence.""" - - configured: FutureCursorsConfig = project_config or FutureCursorsConfig() - raw_distance: object | None = model.config.values.get("cursor_future_max_distance") - raw_action: object | None = model.config.values.get("cursor_future_action") - max_distance: str | None = configured.max_distance - if isinstance(raw_distance, str): - max_distance = None if raw_distance == CURSOR_POLICY_DISABLED else raw_distance - action: FutureCursorAction = configured.action - if isinstance(raw_action, str): - action = FutureCursorAction(raw_action) - return ( - FutureCursorsConfig(max_distance=max_distance, action=action) - if max_distance is not None - else None - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/future_cursor_safety.py b/src/sqlbuild/compiler/planner/_helpers/resolve/future_cursor_safety.py deleted file mode 100644 index ac87304474..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/future_cursor_safety.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Native future cursor safety policy.""" - -from sqlbuild._native import resolve_future_cursor_safety as resolve_future_cursor_safety diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/maximum_start.py b/src/sqlbuild/compiler/planner/_helpers/resolve/maximum_start.py deleted file mode 100644 index 267593051d..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/maximum_start.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Native maximum automatic cursor-start eligibility policy.""" - -from sqlbuild._native import apply_maximum_start_policy as apply_maximum_start_policy -from sqlbuild._native import maximum_allowed_start as maximum_allowed_start diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py deleted file mode 100644 index 8ad3e3c0e5..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Native reference resolution and generic executable-SQL callback support.""" - -from __future__ import annotations - -import re -from collections.abc import Callable, Iterator - -import sqlbuild._native as _native -from sqlbuild._native import apply_deferred_locations as apply_deferred_locations -from sqlbuild._native import build_function_locations as build_function_locations -from sqlbuild._native import build_model_locations as build_model_locations -from sqlbuild._native import build_seed_locations as build_seed_locations -from sqlbuild._native import resolve_dbt_ref_references as resolve_dbt_ref_references -from sqlbuild._native import resolve_ref_references as resolve_ref_references -from sqlbuild._native import resolve_table_function_references as resolve_table_function_references -from sqlbuild._native import resolve_udf_references as resolve_udf_references -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - -_REFERENCE_SCAN_CONTEXT: str = "SQL reference resolution" - - -def replace_executable_matches( - *, - sql: str, - pattern: re.Pattern[str], - replace: Callable[[re.Match[str]], str], - lexical_syntax: SqlLexicalSyntax, -) -> str: - """Replace pattern matches in executable SQL, leaving quoted text and comments untouched.""" - - parts: list[str] = [] - last_index: int = 0 - for match in _iter_code_matches( - sql=sql, - pattern=pattern, - scanner=_native.SqlCodeScanner(sql, lexical_syntax.native_mapping, _REFERENCE_SCAN_CONTEXT), - ): - parts.append(sql[last_index : match.start()]) - parts.append(replace(match)) - last_index = match.end() - parts.append(sql[last_index:]) - return "".join(parts) - - -def _iter_code_matches( - *, sql: str, pattern: re.Pattern[str], scanner: _native.SqlCodeScanner -) -> Iterator[re.Match[str]]: - index: int = 0 - while index < len(sql): - non_code_end: int | None = scanner.non_code_end(index) - if non_code_end is not None: - index = non_code_end - continue - match: re.Match[str] | None = pattern.match(sql, index) - if match is None: - index += 1 - continue - yield match - index = match.end() diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py deleted file mode 100644 index 2802c8ba56..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Native model, function and hook SQL resolution exports.""" - -from sqlbuild._native import resolve_function_sql as resolve_function_sql -from sqlbuild._native import resolve_model_hook_entries as resolve_model_hook_entries -from sqlbuild._native import resolve_model_sql as resolve_model_sql diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py b/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py deleted file mode 100644 index b285a5aa70..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Native source resolution with adapter rendering callbacks.""" - -from sqlbuild._native import render_source_read_relation as render_source_read_relation -from sqlbuild._native import resolve_source_references as resolve_source_references diff --git a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py deleted file mode 100644 index 18cbcfb135..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py +++ /dev/null @@ -1,329 +0,0 @@ -"""Clone planner helper functions.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild._native import ( - build_compiled_function_fingerprint_sql, - build_downstream_deps, - build_execution_upstream_deps, - gather_source_columns, - get_materialization_type, - resolve_graph_selection, - topologically_order_keys, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import ( - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledSourceView, -) -from sqlbuild.compiler.planner._helpers.resolve.refs import ( - build_function_locations, - build_model_locations, - build_seed_locations, -) -from sqlbuild.compiler.planner._helpers.resolve.resolve import ( - resolve_function_sql, - resolve_model_sql, -) -from sqlbuild.compiler.planner.models import ( - BackfillResult, - CloneSourcePlanEntry, - CursorOverridePair, - FunctionPlanEntry, - ModelPlanContext, - ModelPlanEntry, - PlanOutput, - SeedPlanEntry, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - MaterializationType, - PlanAction, - PlanReason, -) -from sqlbuild.spec.contracts.models import SourceEntry - - -def build_clone_plan_output( - *, - project: CompiledProjectView, - select: tuple[str, ...], - exclude: tuple[str, ...], -) -> PlanOutput: - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_execution_upstream_deps(project) - ) - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = build_downstream_deps( - upstream_deps - ) - selected_keys: frozenset[CompiledObjectKey] = resolve_graph_selection( - graph=project.lineage_graph, select=select, exclude=exclude - ) - return PlanOutput( - execution_order=topologically_order_keys(upstream=upstream_deps), - selected_keys=selected_keys, - upstream_deps=upstream_deps, - downstream_deps=downstream_deps, - ) - - -def build_clone_model_entries( - *, - project: CompiledProjectView, - plan: PlanOutput, - adapter: BaseAdapter, - connection: Any, - source_project: CompiledProjectView | None = None, -) -> tuple[ModelPlanEntry, ...]: - effective_source_project: CompiledProjectView = source_project or project - model_locations: dict[str, CompiledRelationLocation] = build_model_locations(project.models) - seed_locations: dict[str, CompiledRelationLocation] = build_seed_locations(project.seeds) - function_locations: dict[str, CompiledRelationLocation] = build_function_locations( - project.functions - ) - source_map: dict[str, SourceEntry] = { - source.name: source.source_entry for source in effective_source_project.sources - } - source_warehouse_columns: dict[str, tuple[Any, ...]] = gather_source_columns( - project=effective_source_project, - adapter=adapter, - connection=connection, - ) - entries_by_key: dict[CompiledObjectKey, ModelPlanEntry] = {} - model: CompiledModelView - for model in project.models: - if model.key not in plan.selected_keys or is_disabled(model): - continue - materialization_type: MaterializationType = get_materialization_type(model) - resolved_sql: str = "" - if materialization_type == MaterializationType.VIEW: - resolved_sql = resolve_model_sql( - adapter=adapter, - model=model, - snapshot=WarehouseSnapshot(), - context=ModelPlanContext( - model_locations=model_locations, - models_by_name={}, - functions_by_name={function.name: function for function in project.functions}, - seed_locations=seed_locations, - function_locations=function_locations, - source_map=source_map, - source_warehouse_columns=source_warehouse_columns, - star_exclude_keyword=adapter.star_exclude_keyword(), - ), - backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), - full_refresh=False, - cursor_overrides=CursorOverridePair(), - ) - entries_by_key[model.key] = ModelPlanEntry( - key=model.key, - name=model.name, - relative_path=model.relative_path, - materialization_type=materialization_type, - action=( - PlanAction.CREATE_VIEW - if materialization_type == MaterializationType.VIEW - else PlanAction.CREATE_TABLE - ), - reason=PlanReason.NO_CHANGE, - destination=model.destination, - fingerprint_query_sql="", - resolved_sql=resolved_sql, - logical_ddl="", - ) - return tuple(entries_by_key[key] for key in plan.execution_order if key in entries_by_key) - - -def build_clone_source_entries( - *, - project: CompiledProjectView, - plan: PlanOutput, - adapter: BaseAdapter, -) -> tuple[CloneSourcePlanEntry, ...]: - entries_by_key: dict[CompiledObjectKey, CloneSourcePlanEntry] = {} - source: CompiledSourceView - for source in project.sources: - source_entry: SourceEntry = source.source_entry - if ( - source.key not in plan.selected_keys - or not source_entry.managed - or source_entry.loader is None - or source_entry.expression is not None - ): - continue - relation_name: str = source_entry.table or source_entry.name - entries_by_key[source.key] = CloneSourcePlanEntry( - key=source.key, - name=source.name, - destination=CompiledRelationLocation( - database=source_entry.database, - schema=source_entry.schema, - name=relation_name, - qualified_name=adapter.render_qualified_name( - database=source_entry.database, - schema=source_entry.schema, - name=relation_name, - ), - ), - ) - return tuple(entries_by_key[key] for key in plan.execution_order if key in entries_by_key) - - -def build_clone_function_entries( - *, - project: CompiledProjectView, - plan: PlanOutput, - adapter: BaseAdapter, - connection: Any, - source_project: CompiledProjectView | None = None, -) -> tuple[FunctionPlanEntry, ...]: - """Build destination-resolved function definitions for clone recreation.""" - effective_source_project: CompiledProjectView = source_project or project - model_locations: dict[str, CompiledRelationLocation] = build_model_locations(project.models) - seed_locations: dict[str, CompiledRelationLocation] = build_seed_locations(project.seeds) - function_locations: dict[str, CompiledRelationLocation] = build_function_locations( - project.functions - ) - source_map: dict[str, SourceEntry] = { - source.name: source.source_entry for source in effective_source_project.sources - } - source_warehouse_columns: dict[str, tuple[Any, ...]] = gather_source_columns( - project=effective_source_project, - adapter=adapter, - connection=connection, - ) - entries_by_key: dict[CompiledObjectKey, FunctionPlanEntry] = {} - function: CompiledFunctionView - for function in project.functions: - if function.key not in plan.selected_keys: - continue - entries_by_key[function.key] = FunctionPlanEntry( - key=function.key, - name=function.name, - relative_path=function.relative_path, - destination=function.destination, - arguments=function.arguments, - returns=function.returns, - body_sql=resolve_function_sql( - adapter=adapter, - function=function, - model_locations=model_locations, - seed_locations=seed_locations, - function_locations=function_locations, - source_map=source_map, - source_warehouse_columns=source_warehouse_columns, - star_exclude_keyword=adapter.star_exclude_keyword(), - ), - fingerprint_query_sql=build_compiled_function_fingerprint_sql(function), - fingerprint_destination=function.fingerprint_destination, - return_columns=function.return_columns, - language=function.language, - source_file_path=function.source_file_path, - runtime_version=function.runtime_version, - entry_point=function.entry_point, - packages=function.packages, - ) - return tuple(entries_by_key[key] for key in plan.execution_order if key in entries_by_key) - - -def build_origin_source_entries( - *, - project: CompiledProjectView, - selected_names: frozenset[str], - adapter: BaseAdapter, -) -> tuple[CloneSourcePlanEntry, ...]: - selected_keys: frozenset[CompiledObjectKey] = frozenset( - source.key for source in project.sources if source.name in selected_names - ) - return build_clone_source_entries( - project=project, - plan=PlanOutput( - execution_order=tuple(source.key for source in project.sources), - selected_keys=selected_keys, - ), - adapter=adapter, - ) - - -def build_origin_model_entries( - *, - project: CompiledProjectView, - selected_names: frozenset[str], -) -> tuple[ModelPlanEntry, ...]: - entries: list[ModelPlanEntry] = [] - model: CompiledModelView - for model in project.models: - if model.name not in selected_names or is_disabled(model): - continue - materialization_type: MaterializationType = get_materialization_type(model) - entries.append( - ModelPlanEntry( - key=model.key, - name=model.name, - relative_path=model.relative_path, - materialization_type=materialization_type, - action=( - PlanAction.CREATE_VIEW - if materialization_type == MaterializationType.VIEW - else PlanAction.CREATE_TABLE - ), - reason=PlanReason.NO_CHANGE, - destination=model.destination, - fingerprint_query_sql="", - resolved_sql="", - logical_ddl="", - ) - ) - return tuple(entries) - - -def build_clone_seed_entries( - *, - project: CompiledProjectView, - plan: PlanOutput, -) -> tuple[SeedPlanEntry, ...]: - return tuple( - SeedPlanEntry( - key=seed.key, - name=seed.name, - destination=seed.destination, - file_path=seed.seed_file.file_path, - columns=tuple(), - csv_settings=seed.schema_entry.csv_settings, - ) - for seed in project.seeds - if seed.key in plan.selected_keys - ) - - -def build_origin_seed_entries( - *, - project: CompiledProjectView, - selected_names: frozenset[str], -) -> tuple[SeedPlanEntry, ...]: - return tuple( - SeedPlanEntry( - key=seed.key, - name=seed.name, - destination=seed.destination, - file_path=seed.seed_file.file_path, - columns=tuple(), - csv_settings=seed.schema_entry.csv_settings, - ) - for seed in project.seeds - if seed.name in selected_names - ) - - -def is_disabled(model: CompiledModelView) -> bool: - raw: object | None = model.config.values.get("enabled") - return isinstance(raw, bool) and not raw diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py deleted file mode 100644 index df9d703e7e..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Contract-aware formatting of SQL test fixture projections.""" - -from sqlbuild._native import format_redundant_fixture_nulls as format_redundant_fixture_nulls diff --git a/src/sqlbuild/compiler/planner/classes/background_sql_test_planning.py b/src/sqlbuild/compiler/planner/classes/background_sql_test_planning.py deleted file mode 100644 index 661972273d..0000000000 --- a/src/sqlbuild/compiler/planner/classes/background_sql_test_planning.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Plan SQL tests on a worker thread while the planner waits on the warehouse.""" - -from __future__ import annotations - -import threading -from concurrent.futures import Future -from types import TracebackType - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.output.plan_output import plan_selected_sql_tests -from sqlbuild.compiler.planner.models import PlannedSqlTests, SqlTestSelection - - -class BackgroundSqlTestPlanning: - """Start warehouse-independent SQL test planning early and join it before assembly.""" - - def __init__( - self, - *, - project: CompiledProjectView, - adapter: BaseAdapter, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, - enabled: bool, - ) -> None: - self._future: Future[PlannedSqlTests] | None = None - if enabled: - future: Future[PlannedSqlTests] = Future() - self._future = future - threading.Thread( - target=_plan_into, - kwargs={ - "future": future, - "project": project, - "adapter": adapter, - "selected_keys": selected_keys, - "sql_test_selection": sql_test_selection, - }, - name="sqlbuild-test-planning", - daemon=True, - ).start() - self._selected_keys: frozenset[CompiledObjectKey] = selected_keys - self._sql_test_selection: SqlTestSelection = sql_test_selection - - def __enter__(self) -> BackgroundSqlTestPlanning: - return self - - def __exit__( - self, - exception_type: type[BaseException] | None, - exception: BaseException | None, - traceback: TracebackType | None, - ) -> None: - del exception_type, exception, traceback - - def result(self) -> PlannedSqlTests: - """Return planned tests, raising any planning error at the original assembly point.""" - - if self._future is None: - return PlannedSqlTests( - selected_keys=self._selected_keys, sql_test_selection=self._sql_test_selection - ) - return self._future.result() - - -def _plan_into( - *, - future: Future[PlannedSqlTests], - project: CompiledProjectView, - adapter: BaseAdapter, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, -) -> None: - """Run planning on a daemon thread so a failed plan never waits for it at exit.""" - - try: - future.set_result( - plan_selected_sql_tests( - project=project, - adapter=adapter, - selected_keys=selected_keys, - sql_test_selection=sql_test_selection, - ) - ) - except BaseException as error: # noqa: BLE001 - re-raised on the joining thread - future.set_exception(error) diff --git a/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py b/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py deleted file mode 100644 index 16cfc725de..0000000000 --- a/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Compiler-backed safe fixture-null source rewrites.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner._helpers.sql_tests.fixture_formatting import ( - format_redundant_fixture_nulls, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -class FixtureNullAutofix: - """Apply fixture-null rewrites proven safe by planner completion.""" - - @staticmethod - def apply( - *, - files: dict[Path, str], - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - sql_lexical_syntax: SqlLexicalSyntax, - ) -> dict[Path, str]: - """Return contract-safe fixture source rewrites.""" - - return format_redundant_fixture_nulls( - files=files, - project_dir=project_dir, - discovered_inputs=discovered_inputs, - sql_lexical_syntax=sql_lexical_syntax, - ) diff --git a/src/sqlbuild/compiler/planner/classes/known_input_columns.py b/src/sqlbuild/compiler/planner/classes/known_input_columns.py deleted file mode 100644 index e8d30db508..0000000000 --- a/src/sqlbuild/compiler/planner/classes/known_input_columns.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Physical columns of the relations a model reads, as inspected by the planner.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.planner.constants import REF_INPUT_FUNCTION, SOURCE_INPUT_FUNCTION - - -class KnownInputColumns: - """Look up the lower-cased physical columns a query binds against, or None when unknown.""" - - def __init__( - self, - *, - model_columns: Mapping[str, tuple[ColumnInfo, ...]], - source_columns: Mapping[str, tuple[ColumnInfo, ...]], - ) -> None: - self._inspected: dict[str, Mapping[str, tuple[ColumnInfo, ...]]] = { - REF_INPUT_FUNCTION: model_columns, - SOURCE_INPUT_FUNCTION: source_columns, - } - - def __call__(self, kind: str, name: str) -> frozenset[str] | None: - """Return the inspected columns of one __source or __ref relation, or None.""" - - columns: tuple[ColumnInfo, ...] = self._inspected.get(kind, {}).get(name, ()) - if not columns: - return None - return frozenset(column.name.lower() for column in columns) diff --git a/src/sqlbuild/compiler/planner/main/__init__.py b/src/sqlbuild/compiler/planner/main/__init__.py deleted file mode 100644 index 1f8ed9be06..0000000000 --- a/src/sqlbuild/compiler/planner/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Planner main package.""" diff --git a/src/sqlbuild/compiler/planner/main/changes/__init__.py b/src/sqlbuild/compiler/planner/main/changes/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/main/changes/_query_diff.py b/src/sqlbuild/compiler/planner/main/changes/_query_diff.py deleted file mode 100644 index 2bc4a905ce..0000000000 --- a/src/sqlbuild/compiler/planner/main/changes/_query_diff.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Public SQL/text diff formatting entrypoint.""" - -from __future__ import annotations - -import difflib - -from sqlbuild.compiler.planner.constants import ( - UNIFIED_DIFF_ADDITION_HEADER_PREFIX, - UNIFIED_DIFF_ADDITION_PREFIX, - UNIFIED_DIFF_REMOVAL_HEADER_PREFIX, - UNIFIED_DIFF_REMOVAL_PREFIX, -) -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def format_query_diff(*, previous: str, current: str) -> list[str]: - """Format a unified diff between previous and current SQL/text.""" - - style: CliStyle = CliStyle(use_color=True) - previous_lines: list[str] = previous.splitlines(keepends=True) - current_lines: list[str] = current.splitlines(keepends=True) - diff_lines: list[str] = list( - difflib.unified_diff(previous_lines, current_lines, fromfile="previous", tofile="current") - ) - result: list[str] = [] - line: str - for line in diff_lines: - stripped: str = line.rstrip("\n") - formatted: str = f" {stripped}" - if ( - stripped[:1] == UNIFIED_DIFF_ADDITION_PREFIX - and stripped[:3] != UNIFIED_DIFF_ADDITION_HEADER_PREFIX - ): - result.append(style.success(formatted)) - elif ( - stripped[:1] == UNIFIED_DIFF_REMOVAL_PREFIX - and stripped[:3] != UNIFIED_DIFF_REMOVAL_HEADER_PREFIX - ): - result.append(style.error(formatted)) - elif stripped.startswith(("---", "+++", "@@")): - result.append(style.muted(formatted)) - else: - result.append(formatted) - return result diff --git a/src/sqlbuild/compiler/planner/main/clone/__init__.py b/src/sqlbuild/compiler/planner/main/clone/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/main/clone/_clone.py b/src/sqlbuild/compiler/planner/main/clone/_clone.py deleted file mode 100644 index 795e3fcda5..0000000000 --- a/src/sqlbuild/compiler/planner/main/clone/_clone.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Public clone planning entrypoints.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.reuse.clone import ( - build_clone_function_entries, - build_clone_model_entries, - build_clone_plan_output, - build_clone_seed_entries, - build_clone_source_entries, - build_origin_model_entries, - build_origin_seed_entries, - build_origin_source_entries, -) -from sqlbuild.compiler.planner.models import ( - CloneSourcePlanEntry, - FunctionPlanEntry, - ModelPlanEntry, - PlanOutput, - SeedPlanEntry, -) - - -def run_clone_planning( - *, - project: CompiledProjectView, - select: tuple[str, ...], - exclude: tuple[str, ...], - adapter: BaseAdapter, - connection: Any, - origin_project: CompiledProjectView, - origin_source_project: CompiledProjectView, - destination_source_project: CompiledProjectView, -) -> tuple[ - PlanOutput, - tuple[CloneSourcePlanEntry, ...], - tuple[ModelPlanEntry, ...], - tuple[SeedPlanEntry, ...], - tuple[FunctionPlanEntry, ...], - tuple[ModelPlanEntry, ...], - tuple[SeedPlanEntry, ...], - tuple[CloneSourcePlanEntry, ...], -]: - """Prepare planner outputs needed for clone execution.""" - - clone_plan: PlanOutput = build_clone_plan_output( - project=project, - select=select, - exclude=exclude, - ) - destination_model_entries: tuple[ModelPlanEntry, ...] = build_clone_model_entries( - project=project, - plan=clone_plan, - adapter=adapter, - connection=connection, - source_project=destination_source_project, - ) - destination_seed_entries: tuple[SeedPlanEntry, ...] = build_clone_seed_entries( - project=project, - plan=clone_plan, - ) - destination_function_entries: tuple[FunctionPlanEntry, ...] = build_clone_function_entries( - project=project, - plan=clone_plan, - adapter=adapter, - connection=connection, - source_project=destination_source_project, - ) - destination_source_entries: tuple[CloneSourcePlanEntry, ...] = () - if destination_source_project.effective_target_name == project.effective_target_name: - destination_source_entries = build_clone_source_entries( - project=project, - plan=clone_plan, - adapter=adapter, - ) - origin_model_entries: tuple[ModelPlanEntry, ...] = build_origin_model_entries( - project=origin_project, - selected_names=frozenset(entry.name for entry in destination_model_entries), - ) - origin_seed_entries: tuple[SeedPlanEntry, ...] = build_origin_seed_entries( - project=origin_project, - selected_names=frozenset(entry.name for entry in destination_seed_entries), - ) - origin_source_entries: tuple[CloneSourcePlanEntry, ...] = build_origin_source_entries( - project=origin_source_project, - selected_names=frozenset(entry.name for entry in destination_source_entries), - adapter=adapter, - ) - origin_source_entries_by_name: dict[str, CloneSourcePlanEntry] = { - entry.name: entry for entry in origin_source_entries - } - destination_source_entries = tuple( - entry - for entry in destination_source_entries - if (origin_entry := origin_source_entries_by_name.get(entry.name)) is not None - and origin_entry.destination != entry.destination - ) - origin_source_entries = tuple( - origin_source_entries_by_name[entry.name] for entry in destination_source_entries - ) - return ( - clone_plan, - destination_source_entries, - destination_model_entries, - destination_seed_entries, - destination_function_entries, - origin_model_entries, - origin_seed_entries, - origin_source_entries, - ) diff --git a/src/sqlbuild/compiler/planner/main/clone/resolve_clone_boundary.py b/src/sqlbuild/compiler/planner/main/clone/resolve_clone_boundary.py deleted file mode 100644 index b9868c49fe..0000000000 --- a/src/sqlbuild/compiler/planner/main/clone/resolve_clone_boundary.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Deferred clone boundary entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable, Mapping - -from sqlbuild.compiler.planner._helpers.graph.defer_clone import resolve_clone_boundary_impl - - -def resolve_clone_boundary[K]( - *, - selected: frozenset[K], - upstream: Mapping[K, tuple[K, ...]], - is_clonable: Callable[[K], bool], - is_view: Callable[[K], bool], -) -> frozenset[K]: - """Return non-view clonable ancestors anchoring selected nodes.""" - - return resolve_clone_boundary_impl( - selected=selected, - upstream=upstream, - is_clonable=is_clonable, - is_view=is_view, - ) diff --git a/src/sqlbuild/compiler/planner/main/clone/resolve_skipped_view_chain.py b/src/sqlbuild/compiler/planner/main/clone/resolve_skipped_view_chain.py deleted file mode 100644 index 75e37c97b5..0000000000 --- a/src/sqlbuild/compiler/planner/main/clone/resolve_skipped_view_chain.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Deferred clone skipped-view entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable, Mapping - -from sqlbuild.compiler.planner._helpers.graph.defer_clone import resolve_skipped_view_chain_impl - - -def resolve_skipped_view_chain[K]( - *, - selected: frozenset[K], - upstream: Mapping[K, tuple[K, ...]], - is_clonable: Callable[[K], bool], - is_view: Callable[[K], bool], -) -> frozenset[K]: - """Return skipped view ancestors rebuilt over cloned boundaries.""" - - return resolve_skipped_view_chain_impl( - selected=selected, - upstream=upstream, - is_clonable=is_clonable, - is_view=is_view, - ) diff --git a/src/sqlbuild/compiler/planner/main/commands/__init__.py b/src/sqlbuild/compiler/planner/main/commands/__init__.py deleted file mode 100644 index 2d27e6e963..0000000000 --- a/src/sqlbuild/compiler/planner/main/commands/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Focused command planning entrypoints.""" diff --git a/src/sqlbuild/compiler/planner/main/commands/_audit.py b/src/sqlbuild/compiler/planner/main/commands/_audit.py deleted file mode 100644 index 79e4b27482..0000000000 --- a/src/sqlbuild/compiler/planner/main/commands/_audit.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public focused audit-plan entrypoint.""" - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.command_planning.static import build_audit_command_plan_impl -from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope, PlanOutput - - -def build_audit_command_plan( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - relations: PlannerRelationsContext, -) -> PlanOutput: - """Project selected audits from canonical static state.""" - - return build_audit_command_plan_impl( - project=project, adapter=adapter, scope=scope, relations=relations - ) diff --git a/src/sqlbuild/compiler/planner/main/commands/_relation_plan.py b/src/sqlbuild/compiler/planner/main/commands/_relation_plan.py deleted file mode 100644 index d622979f2c..0000000000 --- a/src/sqlbuild/compiler/planner/main/commands/_relation_plan.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Public relation-only plan entrypoint.""" - -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.command_planning.static import ( - build_relation_command_plan_impl, -) -from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope, PlanOutput - - -def build_relation_command_plan( - *, project: CompiledProjectView, scope: PlannerScope, relations: PlannerRelationsContext -) -> PlanOutput: - """Project static relation data for Python checks.""" - - return build_relation_command_plan_impl(project=project, scope=scope, relations=relations) diff --git a/src/sqlbuild/compiler/planner/main/commands/_relations.py b/src/sqlbuild/compiler/planner/main/commands/_relations.py deleted file mode 100644 index bf8fd7840e..0000000000 --- a/src/sqlbuild/compiler/planner/main/commands/_relations.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Focused-command relation entrypoint.""" - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.command_planning.static import ( - resolve_static_relation_context_impl, -) -from sqlbuild.compiler.planner.models import DeferralInputs, PlannerRelationsContext, PlannerScope -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -def resolve_static_relation_context( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - deferral: DeferralInputs | None = None, - project_config: ProjectConfig | None = None, - local_config: LocalConfig | None = None, - relation_keys: frozenset[CompiledObjectKey] | None = None, -) -> PlannerRelationsContext: - """Resolve canonical locations and source routing without warehouse inventory.""" - - return resolve_static_relation_context_impl( - project=project, - adapter=adapter, - scope=scope, - deferral=deferral, - project_config=project_config, - local_config=local_config, - relation_keys=relation_keys, - ) diff --git a/src/sqlbuild/compiler/planner/main/commands/_scope.py b/src/sqlbuild/compiler/planner/main/commands/_scope.py deleted file mode 100644 index e2b3c221f1..0000000000 --- a/src/sqlbuild/compiler/planner/main/commands/_scope.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Focused-command scope entrypoint.""" - -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.command_planning.static import ( - resolve_static_command_scope_impl, -) -from sqlbuild.compiler.planner.models import PlannerScope, PlannerSelection - - -def resolve_static_command_scope( - *, project: CompiledProjectView, selection: PlannerSelection, auto_load_sources: bool = False -) -> PlannerScope: - """Resolve command selection through the canonical build selector phase.""" - - return resolve_static_command_scope_impl( - project=project, selection=selection, auto_load_sources=auto_load_sources - ) diff --git a/src/sqlbuild/compiler/planner/main/commands/_seed.py b/src/sqlbuild/compiler/planner/main/commands/_seed.py deleted file mode 100644 index 5f6aee03b2..0000000000 --- a/src/sqlbuild/compiler/planner/main/commands/_seed.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public focused seed-plan entrypoint.""" - -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.command_planning.static import build_seed_command_plan_impl -from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope, PlanOutput - - -def build_seed_command_plan( - *, - project: CompiledProjectView, - scope: PlannerScope, - relations: PlannerRelationsContext, - fingerprints: dict[str, Fingerprint] | None = None, -) -> PlanOutput: - """Project selected direct seed work and canonical identities.""" - - return build_seed_command_plan_impl( - project=project, scope=scope, relations=relations, fingerprints=fingerprints - ) diff --git a/src/sqlbuild/compiler/planner/main/commands/_seed_state.py b/src/sqlbuild/compiler/planner/main/commands/_seed_state.py deleted file mode 100644 index 4c13face97..0000000000 --- a/src/sqlbuild/compiler/planner/main/commands/_seed_state.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public targeted seed-state read entrypoint.""" - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledSeedView -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.command_planning.seed_state import read_seed_fingerprints - - -def read_selected_seed_fingerprints( - *, adapter: BaseAdapter, connection: Any, seeds: tuple[CompiledSeedView, ...] -) -> dict[str, Fingerprint]: - """Read only fingerprint state required by selected direct seeds.""" - - return read_seed_fingerprints(adapter=adapter, connection=connection, seeds=seeds) diff --git a/src/sqlbuild/compiler/planner/main/commands/_sql_test.py b/src/sqlbuild/compiler/planner/main/commands/_sql_test.py deleted file mode 100644 index 089f16569a..0000000000 --- a/src/sqlbuild/compiler/planner/main/commands/_sql_test.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public focused SQL-test plan entrypoint.""" - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.command_planning.static import build_test_command_plan_impl -from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope, PlanOutput - - -def build_test_command_plan( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - relations: PlannerRelationsContext, - case_name: str | None = None, -) -> PlanOutput: - """Project selected SQL tests from canonical static state.""" - - return build_test_command_plan_impl( - project=project, - adapter=adapter, - scope=scope, - relations=relations, - case_name=case_name, - ) diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/__init__.py b/src/sqlbuild/compiler/planner/main/cursor_window/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/_typed_cursor_overrides.py b/src/sqlbuild/compiler/planner/main/cursor_window/_typed_cursor_overrides.py deleted file mode 100644 index 68493db0bd..0000000000 --- a/src/sqlbuild/compiler/planner/main/cursor_window/_typed_cursor_overrides.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Public typing of untyped `--start-cursor`/`--end-cursor` values.""" - -from __future__ import annotations - -from collections.abc import Iterable - -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - type_cursor_overrides, -) -from sqlbuild.compiler.planner.models import CursorOverrides - - -def typed_cursor_overrides( - *, - cursor_overrides: CursorOverrides | None, - selected_models: Iterable[CompiledModelView] = (), -) -> CursorOverrides | None: - """Read `--start-cursor`/`--end-cursor` as the one cursor type of the selected models.""" - - return type_cursor_overrides(cursor_overrides=cursor_overrides, selected_models=selected_models) diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/cursor_window_override_flags.py b/src/sqlbuild/compiler/planner/main/cursor_window/cursor_window_override_flags.py deleted file mode 100644 index 132826c442..0000000000 --- a/src/sqlbuild/compiler/planner/main/cursor_window/cursor_window_override_flags.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public naming of the CLI flags that set a cursor window explicitly.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - cursor_window_override_flags as _cursor_window_override_flags, -) - - -def cursor_window_override_flags(*, cursor_type: str | None) -> str: - """Name the CLI flags that set a cursor window explicitly for this cursor type.""" - - return _cursor_window_override_flags(cursor_type=cursor_type) diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_input_decision.py b/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_input_decision.py deleted file mode 100644 index 543483c310..0000000000 --- a/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_input_decision.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public decision on whether empty cursor inputs leave a model with no window.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - decide_empty_cursor_inputs as _decide_empty_cursor_inputs, -) -from sqlbuild.compiler.planner.models import EmptyCursorInputDecision - - -def decide_empty_cursor_inputs( - *, - empty_input_names: tuple[str, ...], - waits_for_every_input: bool, - inputs_with_rows: bool, - has_window_start: bool, -) -> EmptyCursorInputDecision: - """Decide whether empty cursor inputs leave no window, shared by planning and runtime.""" - - return _decide_empty_cursor_inputs( - empty_input_names=empty_input_names, - waits_for_every_input=waits_for_every_input, - inputs_with_rows=inputs_with_rows, - has_window_start=has_window_start, - ) diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_window.py b/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_window.py deleted file mode 100644 index 4cc7058cf2..0000000000 --- a/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_window.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public zero-width cursor window entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - empty_cursor_window as _empty_cursor_window, -) -from sqlbuild.compiler.planner.models import CursorBounds - - -def empty_cursor_window(*, cursor_type: str | None, cursor_start: str | None) -> CursorBounds: - """Return a zero-width cursor window, which selects no rows from any filtered input.""" - - return _empty_cursor_window(cursor_type=cursor_type, cursor_start=cursor_start) diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/empty_input_rebuild_refusal.py b/src/sqlbuild/compiler/planner/main/cursor_window/empty_input_rebuild_refusal.py deleted file mode 100644 index fae1a78827..0000000000 --- a/src/sqlbuild/compiler/planner/main/cursor_window/empty_input_rebuild_refusal.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Public wording for refusing to rebuild an existing table from empty cursor inputs.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - empty_input_rebuild_refusal as _empty_input_rebuild_refusal, -) - - -def empty_input_rebuild_refusal( - *, model_name: str, input_names: tuple[str, ...], cursor_type: str | None -) -> tuple[str, str]: - """Return the message and help for refusing to rebuild an existing table from empty inputs.""" - - return _empty_input_rebuild_refusal( - model_name=model_name, input_names=input_names, cursor_type=cursor_type - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/__init__.py b/src/sqlbuild/compiler/planner/main/execution/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py b/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py deleted file mode 100644 index 62e4997dad..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Planner entrypoint for SQL-test artifacts planned natively from the compiled objects.""" - -from sqlbuild._native import ( - plan_compiled_sql_test_artifacts as _plan_compiled_sql_test_artifacts, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledSqlTest -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import NativeSqlTestArtifact - - -def plan_compiled_sql_test_artifacts( - *, - project: CompiledProjectView, - tests: tuple[CompiledSqlTest, ...], - adapter: BaseAdapter, - sql_analysis_enabled: bool, -) -> tuple[NativeSqlTestArtifact, ...]: - """Plan and render SQL tests natively from the compiled objects, without a JSON request.""" - - return _plan_compiled_sql_test_artifacts( - project=project, - tests=tests, - adapter=adapter, - sql_analysis_enabled=sql_analysis_enabled, - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/cursor_replay_policy.py b/src/sqlbuild/compiler/planner/main/execution/cursor_replay_policy.py deleted file mode 100644 index b239ea88bc..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/cursor_replay_policy.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Public typed cursor replay policy entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - apply_typed_cursor_replay_policy as _apply_typed_cursor_replay_policy, -) -from sqlbuild.compiler.planner.models import Duration -from sqlbuild.cursor_algebra.types import CursorScalar - - -def apply_typed_cursor_replay_policy( - *, - start: CursorScalar, - end: CursorScalar, - cursor_start: CursorScalar | str | None, - cursor_type: str | None, - lookback: Duration | str | None, - backfill_duration: Duration | str | None, - has_start_override: bool, -) -> CursorScalar: - """Apply replay policy without crossing a string serialization boundary.""" - - return _apply_typed_cursor_replay_policy( - start=start, - end=end, - cursor_start=cursor_start, - cursor_type=cursor_type, - lookback=lookback, - backfill_duration=backfill_duration, - has_start_override=has_start_override, - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/display_plan.py b/src/sqlbuild/compiler/planner/main/execution/display_plan.py deleted file mode 100644 index 8dab9c5583..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/display_plan.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Display-only planner helpers for external plan surfaces.""" - -from __future__ import annotations - -from sqlbuild._native import ( - build_downstream_deps, - build_execution_upstream_deps, - get_config_str, - get_materialization_type, - resolve_model_full_refresh, -) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, -) -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import BackfillResult, ModelPlanEntry, PlanOutput -from sqlbuild.compiler.planner.types import ( - BackfillAction, - MaterializationType, - PlanAction, - PlanReason, -) - - -def build_display_only_sqlbuild_plan( - *, project: CompiledProjectView, selected_model_names: tuple[str, ...], full_refresh: bool -) -> PlanOutput: - """Build a display-only plan when warehouse planning cannot fully resolve.""" - - selected_names: frozenset[str] = frozenset(selected_model_names) - model_entries: list[ModelPlanEntry] = [] - for model in project.models: - if model.name not in selected_names: - continue - materialization_type: MaterializationType = get_materialization_type(model) - effective_full_refresh: bool = resolve_model_full_refresh( - model=model, - cli_full_refresh=full_refresh, - ) - model_entries.append( - ModelPlanEntry( - key=model.key, - name=model.name, - relative_path=model.relative_path, - materialization_type=materialization_type, - action=_display_action( - materialization_type=materialization_type, - full_refresh=effective_full_refresh, - ), - reason=( - PlanReason.FULL_REFRESH if effective_full_refresh else PlanReason.NO_CHANGE - ), - destination=model.destination, - fingerprint_query_sql=model.query_sql, - resolved_sql=model.query_sql, - logical_ddl="", - incremental_strategy=get_config_str( - values=model.config.values, key="incremental_strategy" - ), - incremental_mode=get_config_str(values=model.config.values, key="incremental_mode"), - cursor_column=get_config_str(values=model.config.values, key="cursor_column"), - cursor_type=get_config_str(values=model.config.values, key="cursor_type"), - backfill=BackfillResult( - action=( - BackfillAction.FULL - if effective_full_refresh - else BackfillAction.FORWARD_ONLY - ) - ), - custom_materialization_name=( - get_config_str(values=model.config.values, key="materialized") - if materialization_type == MaterializationType.CUSTOM - else None - ), - ) - ) - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_execution_upstream_deps(project) - ) - return PlanOutput( - execution_order=tuple(entry.key for entry in model_entries), - model_entries=tuple(model_entries), - selected_keys=frozenset(entry.key for entry in model_entries), - upstream_deps=upstream_deps, - downstream_deps=build_downstream_deps(upstream_deps), - ) - - -def _display_action( - *, materialization_type: MaterializationType, full_refresh: bool = False -) -> PlanAction: - if materialization_type == MaterializationType.VIEW: - return PlanAction.CREATE_VIEW - if materialization_type == MaterializationType.INCREMENTAL: - return PlanAction.CREATE_TABLE if full_refresh else PlanAction.INCREMENTAL_APPEND - if materialization_type == MaterializationType.CUSTOM: - return PlanAction.CUSTOM - return PlanAction.CREATE_TABLE diff --git a/src/sqlbuild/compiler/planner/main/execution/execution.py b/src/sqlbuild/compiler/planner/main/execution/execution.py deleted file mode 100644 index 637548b5ff..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/execution.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Top-level planner orchestration producing an execution plan.""" - -from __future__ import annotations - -import time -from collections.abc import Callable -from dataclasses import replace -from typing import Any - -from sqlbuild._native import ( - assemble_base_plan_output, - build_planner_entry_results, - build_planner_identity_context, - build_planner_source_freshness_result, - check_selected_scope_buildability, - detect_changes, - detect_stale_warning_changes, - gather_planner_warehouse_state, - prune_planner_execution_scope, - reconcile_execution_changes, - resolve_model_actions, - resolve_planner_scopes, - with_plan_metadata, - with_plan_warnings, - with_storage_policies, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - type_cursor_overrides, -) -from sqlbuild.compiler.planner.classes.background_sql_test_planning import ( - BackgroundSqlTestPlanning, -) -from sqlbuild.compiler.planner.models import ( - DeferralInputs, - PlannedSqlTests, - PlannerChangeReconciliation, - PlannerChangeResults, - PlannerEntryResults, - PlannerIdentityContext, - PlannerOverrides, - PlannerPolicies, - PlannerResolvedActions, - PlannerRuntime, - PlannerScope, - PlannerScopePruningResult, - PlannerScopeResolution, - PlannerSelection, - PlannerWarehouseState, - PlanOutput, - WarehouseSnapshot, -) -from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -def build_execution_plan( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - selection: PlannerSelection, - overrides: PlannerOverrides, - deferral: DeferralInputs, - policies: PlannerPolicies, - on_progress: Callable[[str], None] | None = None, - project_config: ProjectConfig | None = None, - local_config: LocalConfig | None = None, -) -> PlanOutput: - runtime: PlannerRuntime = PlannerRuntime( - project=project, - adapter=adapter, - connection=connection, - project_config=project_config, - local_config=local_config, - on_progress=on_progress, - ) - scopes: PlannerScopeResolution - scopes, overrides = _resolve_scopes_and_cursor_overrides( - project=project, - selection=selection, - policies=policies, - overrides=overrides, - ) - with ( - BackgroundSqlTestPlanning( - project=project, - adapter=adapter, - selected_keys=scopes.inspection_scope.selected_keys, - sql_test_selection=scopes.inspection_scope.sql_test_selection, - enabled=policies.plan_sql_tests, - ) as test_planning, - open_inspection_catalog(adapter=adapter, connection=connection), - ): - warehouse: PlannerWarehouseState = gather_planner_warehouse_state( - runtime=runtime, - scopes=scopes, - overrides=overrides, - deferral=deferral, - ) - plan_start: float = time.monotonic() - identities: PlannerIdentityContext = build_planner_identity_context( - project=project, - scopes=scopes, - include_stale_warning_identities=policies.selection_diagnostics, - ) - check_selected_scope_buildability( - runtime=runtime, - scopes=scopes, - warehouse=warehouse, - deferral=deferral, - ) - changes: PlannerChangeResults - stale_warning_changes: PlannerChangeResults - changes, stale_warning_changes = _detect_planner_change_results( - project=project, - scopes=scopes, - snapshot=warehouse.snapshot, - identities=identities, - overrides=overrides, - policies=policies, - ) - resolved_actions: PlannerResolvedActions = resolve_model_actions( - scope=scopes.inspection_scope, - changes=changes, - ) - source_freshness: DirectSourceFreshnessPlanningResult = ( - build_planner_source_freshness_result( - project=project, - adapter=adapter, - connection=connection, - scope=scopes.inspection_scope, - relations=warehouse.inspection_relations, - freshness_state_schemas=warehouse.snapshot.source_freshness_state_schemas, - ) - ) - pruning: PlannerScopePruningResult = prune_planner_execution_scope( - scopes=scopes, - resolved_actions=resolved_actions, - ) - reconciliation: PlannerChangeReconciliation = reconcile_execution_changes( - warehouse=warehouse, - identities=identities, - pruning=pruning, - changes=changes, - ) - entries: PlannerEntryResults = build_planner_entry_results( - runtime=runtime, - warehouse=warehouse, - identities=identities, - overrides=overrides, - policies=policies, - deferral=deferral, - pruning=pruning, - reconciliation=reconciliation, - source_freshness=source_freshness, - ) - planned_sql_tests: PlannedSqlTests = test_planning.result() - plan_output: PlanOutput = assemble_base_plan_output( - runtime=runtime, - warehouse=warehouse, - identities=identities, - overrides=overrides, - pruning=pruning, - reconciliation=reconciliation, - entries=entries, - source_freshness=source_freshness, - planned_sql_tests=planned_sql_tests, - ) - plan_output = with_storage_policies( - plan_output=plan_output, - runtime=runtime, - warehouse=warehouse, - scopes=scopes, - policies=policies, - ) - plan_output = with_plan_warnings( - runtime=runtime, - scopes=scopes, - warehouse=warehouse, - identities=identities, - stale_warning_changes=stale_warning_changes, - pruning=pruning, - source_freshness=source_freshness, - plan_output=plan_output, - policies=policies, - ) - plan_output = with_plan_metadata( - plan_output=plan_output, - pruning=pruning, - source_freshness=source_freshness, - policies=policies, - ) - if on_progress is not None: - on_progress(f"Generated plan. ({time.monotonic() - plan_start:.2f}s)") - return plan_output - - -def _resolve_scopes_and_cursor_overrides( - *, - project: CompiledProjectView, - selection: PlannerSelection, - policies: PlannerPolicies, - overrides: PlannerOverrides, -) -> tuple[PlannerScopeResolution, PlannerOverrides]: - scopes: PlannerScopeResolution = resolve_planner_scopes( - project=project, - selection=selection, - policies=policies, - ) - selected_scope: PlannerScope = scopes.selected_scope - selected_models: tuple[CompiledModelView, ...] = tuple( - selected_scope.models_by_name[key.name] - for key in selected_scope.selected_keys - if key.resource_type == CompiledResourceType.MODEL - and key.name in selected_scope.models_by_name - ) - return scopes, replace( - overrides, - cursor_overrides=type_cursor_overrides( - cursor_overrides=overrides.cursor_overrides, selected_models=selected_models - ), - ) - - -def _detect_planner_change_results( - *, - project: CompiledProjectView, - scopes: PlannerScopeResolution, - snapshot: WarehouseSnapshot, - identities: PlannerIdentityContext, - overrides: PlannerOverrides, - policies: PlannerPolicies, -) -> tuple[PlannerChangeResults, PlannerChangeResults]: - changes: PlannerChangeResults = detect_changes( - project=project, - scope=scopes.inspection_scope, - snapshot=snapshot, - full_refresh=overrides.full_refresh, - expected_version_hashes=identities.version_identities.model_version_hashes, - expected_metadata_jsons=identities.version_identities.model_metadata_jsons, - ) - if not policies.selection_diagnostics: - return changes, PlannerChangeResults(models={}, functions={}) - return changes, detect_stale_warning_changes( - project=project, - scopes=scopes, - snapshot=snapshot, - identities=identities, - execution_changes=changes, - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/future_cursor_safety.py b/src/sqlbuild/compiler/planner/main/execution/future_cursor_safety.py deleted file mode 100644 index a4458d654d..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/future_cursor_safety.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Public future cursor safety entry point.""" - -from __future__ import annotations - -from datetime import datetime - -from sqlbuild.compiler.planner._helpers.resolve.future_cursor_safety import ( - resolve_future_cursor_safety, -) -from sqlbuild.compiler.planner.models import ( - CursorBounds, - CursorInputEvidence, -) -from sqlbuild.spec.contracts.models import FutureCursorsConfig - - -def apply_future_cursor_safety( - *, - bounds: CursorBounds, - cursor_type: str | None, - cursor_grain: str | None, - config: FutureCursorsConfig | None, - invocation_time: datetime | None, - has_complete_override: bool, - input_evidence: tuple[CursorInputEvidence, ...] = (), -) -> CursorBounds: - """Apply configured future safety to effective cursor bounds.""" - - return resolve_future_cursor_safety( - bounds=bounds, - cursor_type=cursor_type, - cursor_grain=cursor_grain, - config=config, - invocation_time=invocation_time, - has_complete_override=has_complete_override, - input_evidence=input_evidence, - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/future_cursor_warning.py b/src/sqlbuild/compiler/planner/main/execution/future_cursor_warning.py deleted file mode 100644 index 6bd19aadcb..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/future_cursor_warning.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public future cursor warning rendering entry point.""" - -from __future__ import annotations - -from sqlbuild._native import cursor_render as render -from sqlbuild.compiler.planner.models import CursorBounds, FutureCursorSafetyEvidence - - -def future_cursor_cap_warning(bounds: CursorBounds | None) -> str | None: - """Render a prominent warning for applied future cursor policy.""" - - if bounds is None or bounds.future_safety is None: - return None - evidence: FutureCursorSafetyEvidence = bounds.future_safety - return ( - "FUTURE CURSOR CAPPED: discovered bounds " - f"['{render(value=evidence.discovered_start)}', " - f"'{render(value=evidence.discovered_end)}'), applied bounds " - f"['{render(value=evidence.applied_start)}', '{render(value=evidence.applied_end)}')." - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/maximum_start_safety.py b/src/sqlbuild/compiler/planner/main/execution/maximum_start_safety.py deleted file mode 100644 index 8507b0f775..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/maximum_start_safety.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Public maximum automatic-start safety entry points.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.resolve.maximum_start import ( - apply_maximum_start_policy as _apply_maximum_start_policy, -) -from sqlbuild.compiler.planner.models import ( - CursorBounds, - Duration, - MaximumStartPolicyInputs, - ModelCursorSnapshot, -) -from sqlbuild.cursor_algebra.types import CursorScalar - - -def apply_maximum_start_policy( - *, - bounds: CursorBounds, - snapshot: ModelCursorSnapshot, - cursor_type: str | None, - cursor_grain: str | None, - cursor_start: CursorScalar | str | None, - lookback: Duration | str | None, - backfill_duration: Duration | str | None, - policy: MaximumStartPolicyInputs, - has_start_override: bool, -) -> CursorBounds: - """Apply maximum automatic-start safety to cursor bounds.""" - - return _apply_maximum_start_policy( - bounds=bounds, - snapshot=snapshot, - cursor_type=cursor_type, - cursor_grain=cursor_grain, - cursor_start=cursor_start, - lookback=lookback, - backfill_duration=backfill_duration, - policy=policy, - has_start_override=has_start_override, - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/maximum_start_warning.py b/src/sqlbuild/compiler/planner/main/execution/maximum_start_warning.py deleted file mode 100644 index 061ec067fd..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/maximum_start_warning.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Public maximum automatic-start warning rendering entry point.""" - -from __future__ import annotations - -from sqlbuild._native import cursor_render as render -from sqlbuild.compiler.planner.models import CursorBounds, MaximumStartSafetyEvidence -from sqlbuild.cursor_algebra.types import CursorScalar - - -def maximum_start_cap_warning(bounds: CursorBounds | None) -> str | None: - """Render a warning that distinguishes start recovery from future-end capping.""" - - if bounds is None or bounds.maximum_start_safety is None: - return None - evidence: MaximumStartSafetyEvidence = bounds.maximum_start_safety - return ( - "AUTOMATIC START CAPPED: physical target MAX " - f"'{render(value=evidence.physical_target_max)}' exceeded horizon " - f"'{render(value=evidence.maximum_allowed_start)}'; highest eligible target MAX=" - f"{_render_optional_repr(evidence.highest_eligible_target_max)}, " - "effective post-lookback start=" - f"'{render(value=evidence.effective_start)}', action={evidence.action.value}, " - f"input={evidence.target_relation}.{evidence.cursor_column}." - ) - - -def _render_optional_repr(value: CursorScalar | None) -> str: - if value is None: - return "None" - return repr(render(value=value)) diff --git a/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py b/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py deleted file mode 100644 index 4412ed1b5c..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public planner entrypoint for batched SQL-test model closure discovery.""" - -from sqlbuild._native import ( - resolve_sql_test_model_chains, -) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlTest -from sqlbuild.compiler.compile.types import CompiledProjectView - - -def sql_test_model_chain_names_by_key( - *, project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...] -) -> dict[CompiledObjectKey, tuple[str, ...]]: - """Return model closures resolved natively in one project batch.""" - - return { - test.key: chain - for test, chain in zip( - tests, resolve_sql_test_model_chains(project=project, tests=tests), strict=True - ) - } diff --git a/src/sqlbuild/compiler/planner/main/pre_build/__init__.py b/src/sqlbuild/compiler/planner/main/pre_build/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py deleted file mode 100644 index 7915fda271..0000000000 --- a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Plan text for storage and migration work that runs before model builds.""" - -from __future__ import annotations - -from sqlbuild._native import append_overflow_line, format_model_migrations, visible_entries -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.presentation.models import DisplayOptions - - -def format_pre_build_work( - *, lines: list[str], plan: PlanOutput, display_options: DisplayOptions -) -> list[str]: - """Format retention, table-type, and migration work that runs before model builds.""" - - if plan.table_type_entries: - lines.append("Table type conversions") - for table_type_entry in visible_entries( - entries=plan.table_type_entries, options=display_options - ): - actual_type: str = table_type_entry.actual_type or "unknown" - lines.append( - f" {table_type_entry.model_name} desired={table_type_entry.desired_type} " - f"actual={actual_type} source={table_type_entry.source}" - ) - if table_type_entry.irreversible_warning is not None: - lines.append(f" WARNING: {table_type_entry.irreversible_warning}") - lines = append_overflow_line( - lines=lines, - total_count=len(plan.table_type_entries), - visible_count=len( - visible_entries(entries=plan.table_type_entries, options=display_options) - ), - indent=" ", - options=display_options, - ) - if plan.retention_entries: - lines.append("Retention") - for entry in plan.retention_entries: - scope: str = ".".join( - part - for part in (entry.request.database, entry.request.schema, entry.request.name) - if part - ) - actual: str = "missing" if entry.actual_days is None else f"{entry.actual_days}d" - effective: str = "missing" if entry.effective_days is None else f"{entry.effective_days}d" - lines.append( - f" {scope} desired={entry.request.desired_days}d actual={actual} " - f"effective={effective} source={entry.source} direction={entry.direction.value} " - f"phase={entry.phase.value}" - ) - if entry.irreversible_warning is not None: - lines.append(f" WARNING: {entry.irreversible_warning}") - return format_model_migrations(lines=lines, plan=plan, display_options=display_options) diff --git a/src/sqlbuild/compiler/planner/main/scenarios/__init__.py b/src/sqlbuild/compiler/planner/main/scenarios/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/main/scenarios/_build_scenario_artifact_physical_name.py b/src/sqlbuild/compiler/planner/main/scenarios/_build_scenario_artifact_physical_name.py deleted file mode 100644 index f1e0241e81..0000000000 --- a/src/sqlbuild/compiler/planner/main/scenarios/_build_scenario_artifact_physical_name.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public scenario artifact physical-name builder entrypoint.""" - -from __future__ import annotations - -from sqlbuild._native import ( - build_scenario_artifact_physical_name as _build_scenario_artifact_physical_name, -) - - -def build_scenario_artifact_physical_name( - *, hash_prefix: str, kind: str, logical_name: str, identifier_limit: int -) -> str: - """Build one deterministic scenario artifact physical relation name.""" - - return _build_scenario_artifact_physical_name( - hash_prefix=hash_prefix, - kind=kind, - logical_name=logical_name, - identifier_limit=identifier_limit, - ) diff --git a/src/sqlbuild/compiler/planner/main/scenarios/is_scenario_artifact_physical_name.py b/src/sqlbuild/compiler/planner/main/scenarios/is_scenario_artifact_physical_name.py deleted file mode 100644 index bced73f581..0000000000 --- a/src/sqlbuild/compiler/planner/main/scenarios/is_scenario_artifact_physical_name.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public scenario artifact physical-name detection entrypoint.""" - -from __future__ import annotations - -from sqlbuild._native import ( - is_scenario_artifact_physical_name as _is_scenario_artifact_physical_name, -) - - -def is_scenario_artifact_physical_name(name: str) -> bool: - """Return whether a relation name matches SQLBuild's scenario artifact shape.""" - - return _is_scenario_artifact_physical_name(name) diff --git a/src/sqlbuild/compiler/planner/main/scenarios/scenario.py b/src/sqlbuild/compiler/planner/main/scenarios/scenario.py deleted file mode 100644 index b37b5a2a73..0000000000 --- a/src/sqlbuild/compiler/planner/main/scenarios/scenario.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Public scenario planner entry point.""" - -from __future__ import annotations - -from sqlbuild._native import build_cli_scenario_plan -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -def build_scenario_plan( - *, - scenario: CompiledSqlScenario, - pipeline_result: CompilePipelineResult, - adapter: BaseAdapter, - project_name: str, - source_lexical_syntax: SqlLexicalSyntax, -) -> ScenarioExecutionPlan: - """Build a scenario execution plan for an external entrypoint.""" - - return build_cli_scenario_plan( - scenario=scenario, - pipeline_result=pipeline_result, - adapter=adapter, - project_name=project_name, - source_lexical_syntax=source_lexical_syntax, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/__init__.py b/src/sqlbuild/compiler/planner/main/selection/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/main/selection/_resolve_planner_scopes.py b/src/sqlbuild/compiler/planner/main/selection/_resolve_planner_scopes.py deleted file mode 100644 index c1f60c120f..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_resolve_planner_scopes.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Planner scope resolution entrypoint for compiler orchestration.""" - -from __future__ import annotations - -from sqlbuild._native import resolve_planner_scopes as _resolve_planner_scopes -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import ( - PlannerPolicies, - PlannerScopeResolution, - PlannerSelection, -) - - -def resolve_planner_scopes( - *, - project: CompiledProjectView, - selection: PlannerSelection, - policies: PlannerPolicies, -) -> PlannerScopeResolution: - """Resolve selected, stale-warning, and inspection scopes for one plan.""" - - return _resolve_planner_scopes( - project=project, - selection=selection, - policies=policies, - ) diff --git a/src/sqlbuild/compiler/project_assembly/main/__init__.py b/src/sqlbuild/compiler/project_assembly/main/__init__.py deleted file mode 100644 index 646fd16ff4..0000000000 --- a/src/sqlbuild/compiler/project_assembly/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native compiled project assembly entry points.""" diff --git a/src/sqlbuild/compiler/project_assembly/main/_check_native_sql_syntax.py b/src/sqlbuild/compiler/project_assembly/main/_check_native_sql_syntax.py deleted file mode 100644 index 0a00801549..0000000000 --- a/src/sqlbuild/compiler/project_assembly/main/_check_native_sql_syntax.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Check SQL syntax natively.""" - -from __future__ import annotations - -import sqlbuild._native as _native - - -def check_native_sql_syntax( - *, checks: tuple[tuple[str, dict[str, str] | None], ...], dialect: str | None -) -> bool: - """Whether every `(sql, placeholders)` parses; raises Python's pre-parse `ValueError`.""" - - return _native.check_native_sql_syntax( - ( - dialect or "generic", - [(sql, list((placeholders or {}).items())) for sql, placeholders in checks], - ) - ) diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/__init__.py b/src/sqlbuild/compiler/source_freshness/_helpers/__init__.py deleted file mode 100644 index cf805ebda2..0000000000 --- a/src/sqlbuild/compiler/source_freshness/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Shared source freshness helper implementations.""" diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/datetime.py b/src/sqlbuild/compiler/source_freshness/_helpers/datetime.py deleted file mode 100644 index 6ad95d119f..0000000000 --- a/src/sqlbuild/compiler/source_freshness/_helpers/datetime.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Source freshness datetime invariant helpers.""" - -from __future__ import annotations - -from datetime import UTC, datetime - -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessObservationError - - -def require_aware_utc_datetime(*, value: datetime, field_name: str) -> datetime: - """Require an aware freshness datetime and normalize it to UTC.""" - - if value.tzinfo is None or value.utcoffset() is None: - raise SourceFreshnessObservationError( - f"source freshness {field_name} must be timezone-aware" - ) - return value.astimezone(UTC) - - -def normalize_presumed_utc_datetime(*, value: datetime) -> datetime: - """Treat a naive adapter or legacy-state datetime as UTC and normalize aware values.""" - - if value.tzinfo is None or value.utcoffset() is None: - return value.replace(tzinfo=UTC) - return value.astimezone(UTC) diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/observation.py b/src/sqlbuild/compiler/source_freshness/_helpers/observation.py deleted file mode 100644 index 3d309d31b7..0000000000 --- a/src/sqlbuild/compiler/source_freshness/_helpers/observation.py +++ /dev/null @@ -1,323 +0,0 @@ -"""Shared source freshness observation helper implementation.""" - -from __future__ import annotations - -from datetime import datetime -from typing import Any - -from sqlbuild._native import render_source_relation -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.adapter.contract.models import ( - QueryResult, - TableFreshnessMetadata, - TableFreshnessRequest, -) -from sqlbuild.adapter.contract.types import TableFreshnessStatus -from sqlbuild.compiler.source_freshness._helpers.datetime import normalize_presumed_utc_datetime -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessObservationError -from sqlbuild.compiler.source_freshness.models import ( - AdapterSourceFreshnessBatch, - SourceFreshnessObservation, - SourceFreshnessUnknown, -) -from sqlbuild.compiler.source_freshness.types import SourceFreshnessUnknownReason -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.runtime.observability.models import OperationAttributes -from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind - - -def observe_configured_source_freshness( - *, - adapter: StrictAdapter, - connection: Any, - source: SourceEntry, - observed_at: datetime, -) -> SourceFreshnessObservation: - """Observe one source freshness config and return a comparable data version.""" - - config: SourceFreshnessConfig | None = source.freshness - strategy: str = config.strategy.value if config is not None else "sql" - operation_name: str = ( - "source_freshness_metadata_observation" - if config is not None and config.strategy == SourceFreshnessStrategy.ADAPTER - else "source_freshness_query_observation" - ) - with OperationLifecycle( - operation_kind="freshness", - operation_name=operation_name, - attributes=OperationAttributes(phase="observe", strategy=strategy, target_kind="source"), - ): - return _observe_configured_source_freshness( - adapter=adapter, - connection=connection, - source=source, - observed_at=observed_at, - ) - - -def _observe_configured_source_freshness( - *, - adapter: StrictAdapter, - connection: Any, - source: SourceEntry, - observed_at: datetime, -) -> SourceFreshnessObservation: - config: SourceFreshnessConfig | None = source.freshness - if config is None: - raise SourceFreshnessObservationError( - f"source '{source.name}' does not define freshness configuration" - ) - if config.strategy == SourceFreshnessStrategy.ADAPTER: - return _observe_adapter_freshness( - adapter=adapter, - connection=connection, - source=source, - observed_at=observed_at, - ) - - if config.strategy == SourceFreshnessStrategy.COLUMN: - if config.column is None or config.value_kind is None: - raise SourceFreshnessObservationError( - f"source '{source.name}' has incomplete column freshness configuration" - ) - where_sql: str = f" WHERE {config.filter}" if config.filter is not None else "" - sql: str = adapter.render_source_freshness_max_query( - column=config.column, - source_relation=render_source_relation(entry=source, adapter=adapter), - source_is_subquery=source.expression is not None, - where_sql=where_sql, - ) - data_version: object = _normalize_adapter_datetime( - _query_single_data_version( - adapter=adapter, - connection=connection, - source_name=source.name, - sql=sql, - ) - ) - return SourceFreshnessObservation( - source_name=source.name, - strategy=config.strategy, - data_version=data_version, - value_kind=config.value_kind, - observed_at=observed_at, - ) - - if config.query is None or config.value_kind is None: - raise SourceFreshnessObservationError( - f"source '{source.name}' has incomplete sql freshness configuration" - ) - data_version = _normalize_adapter_datetime( - _query_single_data_version( - adapter=adapter, - connection=connection, - source_name=source.name, - sql=config.query, - ) - ) - return SourceFreshnessObservation( - source_name=source.name, - strategy=config.strategy, - data_version=data_version, - value_kind=config.value_kind, - observed_at=observed_at, - ) - - -def _observe_adapter_freshness( - *, - adapter: StrictAdapter, - connection: Any, - source: SourceEntry, - observed_at: datetime, -) -> SourceFreshnessObservation: - if not adapter.supports_table_freshness_metadata(): - raise SourceFreshnessObservationError( - f"adapter '{adapter.adapter_name}' does not support table freshness metadata" - ) - if source.expression is not None or source.table is None: - raise SourceFreshnessObservationError( - f"source '{source.name}' adapter freshness requires a physical table source" - ) - metadata: TableFreshnessMetadata = adapter.get_table_freshness_metadata( - connection=connection, - database=source.database, - schema=source.schema, - name=source.table, - ) - source_unknown: SourceFreshnessUnknown | None = _adapter_metadata_unknown( - source_name=source.name, metadata=metadata - ) - if source_unknown is not None: - raise SourceFreshnessObservationError(source_unknown.message) - return SourceFreshnessObservation( - source_name=source.name, - strategy=SourceFreshnessStrategy.ADAPTER, - data_version=_normalize_adapter_datetime(metadata.data_version), - value_kind=SourceFreshnessValueKind(metadata.value_kind), - observed_at=( - normalize_presumed_utc_datetime(value=metadata.observed_at) - if metadata.observed_at is not None - else observed_at - ), - ) - - -def observe_adapter_sources_freshness( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - observed_at: datetime, -) -> AdapterSourceFreshnessBatch: - """Observe adapter metadata freshness per source, batching the warehouse lookups.""" - - if not sources: - return AdapterSourceFreshnessBatch() - with OperationLifecycle( - operation_kind="freshness", - operation_name="source_freshness_metadata_observation", - metadata={"item_count": len(sources)}, - attributes=OperationAttributes(phase="observe", strategy="adapter", target_kind="source"), - ) as lifecycle: - batch: AdapterSourceFreshnessBatch = _observe_adapter_sources_freshness( - adapter=adapter, - connection=connection, - sources=sources, - observed_at=observed_at, - ) - lifecycle.completed( - metadata={"item_count": len(sources), "unknown_count": len(batch.unknown)} - ) - return batch - - -def _observe_adapter_sources_freshness( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - observed_at: datetime, -) -> AdapterSourceFreshnessBatch: - unknown: dict[str, SourceFreshnessUnknown] = {} - requests_by_source_name: dict[str, TableFreshnessRequest] = {} - source: SourceEntry - for source in sources: - unsupported_message: str | None = _adapter_freshness_unsupported_message( - adapter=adapter, source=source - ) - if unsupported_message is not None: - unknown[source.name] = SourceFreshnessUnknown( - source_name=source.name, - reason=SourceFreshnessUnknownReason.UNAVAILABLE, - message=unsupported_message, - ) - continue - requests_by_source_name[source.name] = TableFreshnessRequest( - database=source.database, - schema=source.schema, - name=str(source.table), - ) - if not requests_by_source_name: - return AdapterSourceFreshnessBatch(unknown=unknown) - - metadata_by_request: dict[TableFreshnessRequest, TableFreshnessMetadata] = ( - adapter.get_tables_freshness_metadata( - connection=connection, - requests=tuple(dict.fromkeys(requests_by_source_name.values())), - ) - ) - observations: dict[str, SourceFreshnessObservation] = {} - source_name: str - request: TableFreshnessRequest - for source_name, request in requests_by_source_name.items(): - metadata: TableFreshnessMetadata = metadata_by_request.get( - request, - TableFreshnessMetadata.unavailable( - message=f"source '{source_name}' freshness metadata was not returned" - ), - ) - source_unknown: SourceFreshnessUnknown | None = _adapter_metadata_unknown( - source_name=source_name, metadata=metadata - ) - if source_unknown is not None: - unknown[source_name] = source_unknown - continue - observations[source_name] = SourceFreshnessObservation( - source_name=source_name, - strategy=SourceFreshnessStrategy.ADAPTER, - data_version=_normalize_adapter_datetime(metadata.data_version), - value_kind=SourceFreshnessValueKind(metadata.value_kind), - observed_at=( - normalize_presumed_utc_datetime(value=metadata.observed_at) - if metadata.observed_at is not None - else observed_at - ), - ) - return AdapterSourceFreshnessBatch(observations=observations, unknown=unknown) - - -def _adapter_freshness_unsupported_message( - *, adapter: StrictAdapter, source: SourceEntry -) -> str | None: - if not adapter.supports_table_freshness_metadata(): - return f"adapter '{adapter.adapter_name}' does not support table freshness metadata" - if source.expression is not None or source.table is None: - return f"source '{source.name}' adapter freshness requires a physical table source" - return None - - -def _adapter_metadata_unknown( - *, source_name: str, metadata: TableFreshnessMetadata -) -> SourceFreshnessUnknown | None: - if metadata.status == TableFreshnessStatus.UNAVAILABLE: - return SourceFreshnessUnknown( - source_name=source_name, - reason=SourceFreshnessUnknownReason.UNAVAILABLE, - message=metadata.message or f"source '{source_name}' freshness metadata unavailable", - ) - if metadata.data_version is None: - return SourceFreshnessUnknown( - source_name=source_name, - reason=SourceFreshnessUnknownReason.UNAVAILABLE, - message=f"source '{source_name}' freshness data_version is null", - ) - return None - - -def _query_single_data_version( - *, - adapter: StrictAdapter, - connection: Any, - source_name: str, - sql: str, -) -> object: - try: - result: QueryResult = adapter.query(connection=connection, sql=sql, limit=None) - except Exception as exc: - raise SourceFreshnessObservationError( - f"source '{source_name}' freshness query failed: {exc}" - ) from exc - if len(result.columns) != 1: - raise SourceFreshnessObservationError( - f"source '{source_name}' freshness query must return exactly one column" - ) - if len(result.rows) != 1: - raise SourceFreshnessObservationError( - f"source '{source_name}' freshness query must return exactly one row" - ) - data_version: object = result.rows[0][0] - if data_version is None: - raise SourceFreshnessObservationError( - f"source '{source_name}' freshness data_version cannot be null" - ) - return data_version - - -def _normalize_adapter_datetime(value: object) -> object: - """Normalize driver datetimes, whose naive values are documented as presumed UTC.""" - - if not isinstance(value, datetime): - return value - return normalize_presumed_utc_datetime(value=value) diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/planning.py b/src/sqlbuild/compiler/source_freshness/_helpers/planning.py deleted file mode 100644 index 3a29acfe20..0000000000 --- a/src/sqlbuild/compiler/source_freshness/_helpers/planning.py +++ /dev/null @@ -1,337 +0,0 @@ -"""Direct source freshness planning observation and comparison helpers.""" - -from __future__ import annotations - -from collections.abc import Callable -from datetime import datetime -from typing import Any - -from sqlbuild._native import ( - evaluate_source_freshness_age_policy, - normalize_source_freshness_data_version, - source_freshness_data_version_hash, - source_freshness_records_equivalent, -) -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.compiler.source_freshness.constants import ( - INCOMPLETE_CONFIGURATION_ERROR_FRAGMENT, - PHYSICAL_TABLE_SOURCE_ERROR_FRAGMENT, -) -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessObservationError -from sqlbuild.compiler.source_freshness.main.adapter_observation import ( - observe_adapter_sources_freshness, -) -from sqlbuild.compiler.source_freshness.main.observation import ( - observe_configured_source_freshness, -) -from sqlbuild.compiler.source_freshness.main.read import read_latest_source_freshness -from sqlbuild.compiler.source_freshness.models import ( - AdapterSourceFreshnessBatch, - DirectSourceFreshnessPlanningResult, - SourceFreshnessIdentity, - SourceFreshnessObservation, - SourceFreshnessRecord, - SourceFreshnessSet, - SourceFreshnessUnknown, -) -from sqlbuild.compiler.source_freshness.types import ( - SourceFreshnessAgeStatus, - SourceFreshnessUnknownReason, -) -from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy - - -def build_direct_source_freshness_planning_result( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - state_database: str | None, - state_schemas: tuple[str, ...], - observed_at: datetime, - run_id: str, - render_qualified_name: Callable[..., str | None], - state_table_exists_by_schema: dict[str, bool], - filter_previous_to_sources: bool = False, -) -> DirectSourceFreshnessPlanningResult: - previous_records_by_identity: dict[SourceFreshnessIdentity, SourceFreshnessRecord] = ( - _read_previous_records( - adapter=adapter, - connection=connection, - state_database=state_database, - state_schemas=state_schemas, - render_qualified_name=render_qualified_name, - state_table_exists_by_schema=state_table_exists_by_schema, - source_names=( - tuple(sorted(source.name for source in sources)) - if filter_previous_to_sources - else None - ), - ) - ) - observed_records: list[SourceFreshnessRecord] = [] - unconfigured_sources: dict[str, SourceFreshnessUnknown] = {} - changed_identities: set[SourceFreshnessIdentity] = set() - unchanged_identities: set[SourceFreshnessIdentity] = set() - age_statuses: dict[SourceFreshnessIdentity, SourceFreshnessAgeStatus] = {} - observation_sources_by_name: dict[str, SourceEntry] = {} - adapter_observation_sources: list[SourceEntry] = [] - - source: SourceEntry - for source in sources: - if source.managed: - continue - observation_source: SourceEntry | None = _source_for_observation( - adapter=adapter, - source=source, - ) - if observation_source is None: - unconfigured_sources[source.name] = SourceFreshnessUnknown( - source_name=source.name, - reason=SourceFreshnessUnknownReason.NO_FRESHNESS_CONFIG, - message="no freshness config and adapter metadata unavailable", - ) - continue - observation_sources_by_name[source.name] = observation_source - if observation_source.freshness is not None and ( - observation_source.freshness.strategy == SourceFreshnessStrategy.ADAPTER - ): - adapter_observation_sources.append(observation_source) - - adapter_batch: AdapterSourceFreshnessBatch = _observe_adapter_batch( - adapter=adapter, - connection=connection, - sources=tuple(adapter_observation_sources), - observed_at=observed_at, - ) - unknown_sources: dict[str, SourceFreshnessUnknown] = { - **unconfigured_sources, - **adapter_batch.unknown, - } - - for source_name, observation_source in observation_sources_by_name.items(): - if observation_source.freshness is not None and ( - observation_source.freshness.strategy == SourceFreshnessStrategy.ADAPTER - ): - observation: SourceFreshnessObservation | None = adapter_batch.observations.get( - source_name - ) - if observation is None: - continue - else: - try: - observation = observe_configured_source_freshness( - adapter=adapter, - connection=connection, - source=observation_source, - observed_at=observed_at, - ) - except AdapterUserError as exc: - unknown_sources[source_name] = _error_unknown( - source_name=source_name, message=exc.message - ) - continue - except SourceFreshnessObservationError as exc: - if _source_freshness_error_is_configuration_error(error=exc): - raise - unknown_sources[source_name] = _error_unknown( - source_name=source_name, message=str(exc) - ) - continue - observed_record: SourceFreshnessRecord = source_freshness_record_from_observation( - observation=observation, - source=observation_source, - run_id=run_id, - ) - observed_records.append(observed_record) - age_status: SourceFreshnessAgeStatus | None = evaluate_source_freshness_age_policy( - policy=observation_source.freshness.age_policy - if observation_source.freshness is not None - else None, - data_version=observation.data_version, - observed_at=observation.observed_at, - ) - if age_status is not None: - age_statuses[observed_record.identity] = age_status - previous_record: SourceFreshnessRecord | None = previous_records_by_identity.get( - observed_record.identity - ) - if previous_record is None: - changed_identities.add(observed_record.identity) - continue - if source_freshness_records_equivalent( - previous_record=previous_record, - current_record=observed_record, - lag_tolerance=observation_source.freshness.lag_tolerance - if observation_source.freshness is not None - else None, - ): - unchanged_identities.add(observed_record.identity) - else: - changed_identities.add(observed_record.identity) - - return DirectSourceFreshnessPlanningResult( - observed_records=tuple(sorted(observed_records, key=lambda record: str(record.identity))), - previous_records=tuple( - sorted(previous_records_by_identity.values(), key=lambda record: str(record.identity)) - ), - changed_identities=frozenset(changed_identities), - unchanged_identities=frozenset(unchanged_identities), - unknown_source_names=tuple(sorted(unknown_sources)), - unknown_sources=dict(sorted(unknown_sources.items())), - age_statuses=age_statuses, - ) - - -def _observe_adapter_batch( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - observed_at: datetime, -) -> AdapterSourceFreshnessBatch: - try: - return observe_adapter_sources_freshness( - adapter=adapter, - connection=connection, - sources=sources, - observed_at=observed_at, - ) - except (AdapterUserError, SourceFreshnessObservationError) as exc: - message: str = exc.message if isinstance(exc, AdapterUserError) else str(exc) - return AdapterSourceFreshnessBatch( - unknown={ - source.name: _error_unknown(source_name=source.name, message=message) - for source in sources - } - ) - - -def _error_unknown(*, source_name: str, message: str) -> SourceFreshnessUnknown: - return SourceFreshnessUnknown( - source_name=source_name, - reason=SourceFreshnessUnknownReason.ERROR, - message=message, - ) - - -def _source_freshness_error_is_configuration_error( - *, error: SourceFreshnessObservationError -) -> bool: - message: str = str(error) - return ( - PHYSICAL_TABLE_SOURCE_ERROR_FRAGMENT in message - or INCOMPLETE_CONFIGURATION_ERROR_FRAGMENT in message - ) - - -def source_freshness_record_from_observation( - *, observation: SourceFreshnessObservation, source: SourceEntry, run_id: str -) -> SourceFreshnessRecord: - normalized_data_version: str = normalize_source_freshness_data_version( - value=observation.data_version, - value_kind=observation.value_kind, - ) - return SourceFreshnessRecord( - source_name=observation.source_name, - target_database=source.database, - target_schema=source.schema, - target_name=source.table, - run_id=run_id, - strategy=observation.strategy.value, - value_kind=observation.value_kind.value, - data_version=normalized_data_version, - data_version_hash=source_freshness_data_version_hash( - source_name=observation.source_name, - strategy=observation.strategy, - value_kind=observation.value_kind, - data_version=normalized_data_version, - ), - observed_at=observation.observed_at, - ) - - -def _read_previous_records( - *, - adapter: StrictAdapter, - connection: Any, - state_database: str | None, - state_schemas: tuple[str, ...], - render_qualified_name: Callable[..., str | None], - state_table_exists_by_schema: dict[str, bool], - source_names: tuple[str, ...] | None, -) -> dict[SourceFreshnessIdentity, SourceFreshnessRecord]: - previous_records_by_identity: dict[SourceFreshnessIdentity, SourceFreshnessRecord] = {} - state_schema: str - for state_schema in state_schemas: - table_exists: bool = state_table_exists_by_schema.get(state_schema, False) - previous_set: SourceFreshnessSet = read_latest_source_freshness( - connection=connection, - execute=adapter.execute, - table_exists=table_exists, - database=state_database, - schema=state_schema, - render_qualified_name=render_qualified_name, - render_read_latest_sql=adapter.render_read_latest_source_freshness_sql, - source_names=source_names, - ) - previous_records_by_identity = _merged_latest_previous_records( - previous_records_by_identity=previous_records_by_identity, - candidate_records=previous_set.records, - ) - return previous_records_by_identity - - -def _merged_latest_previous_records( - *, - previous_records_by_identity: dict[SourceFreshnessIdentity, SourceFreshnessRecord], - candidate_records: dict[SourceFreshnessIdentity, SourceFreshnessRecord], -) -> dict[SourceFreshnessIdentity, SourceFreshnessRecord]: - merged: dict[SourceFreshnessIdentity, SourceFreshnessRecord] = dict( - previous_records_by_identity - ) - identity: SourceFreshnessIdentity - candidate_record: SourceFreshnessRecord - for identity, candidate_record in candidate_records.items(): - previous_record: SourceFreshnessRecord | None = merged.get(identity) - if previous_record is None or candidate_record.observed_at > previous_record.observed_at: - merged[identity] = candidate_record - return merged - - -def _source_for_observation(*, adapter: StrictAdapter, source: SourceEntry) -> SourceEntry | None: - if source.freshness is not None: - return source - if ( - source.expression is None - and source.table is not None - and adapter.supports_table_freshness_metadata() - ): - return _source_with_adapter_freshness(source) - return None - - -def _source_with_adapter_freshness(source: SourceEntry) -> SourceEntry: - return SourceEntry( - name=source.name, - database=source.database, - schema=source.schema, - table=source.table, - loader=source.loader, - managed=source.managed, - integration_loader=source.integration_loader, - freshness=SourceFreshnessConfig(strategy=SourceFreshnessStrategy.ADAPTER), - write_strategy=source.write_strategy, - load_batch_size=source.load_batch_size, - cursor_column=source.cursor_column, - unique_key=source.unique_key, - expression=source.expression, - description=source.description, - type_enforcement=source.type_enforcement, - contract=source.contract, - meta=source.meta, - columns=source.columns, - audits=source.audits, - ) diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py b/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py deleted file mode 100644 index 9368d38745..0000000000 --- a/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Direct source freshness downstream propagation helpers.""" - -from __future__ import annotations - -from sqlbuild._native import build_downstream_model_name_closure -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import PlannerScope -from sqlbuild.compiler.source_freshness.models import ( - DirectSourceFreshnessPlanningResult, - DirectSourceFreshnessPropagationResult, - SourceFreshnessIdentity, -) -from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus - - -def build_direct_source_freshness_propagation_result( - *, - source_freshness: DirectSourceFreshnessPlanningResult, - scope: PlannerScope, -) -> DirectSourceFreshnessPropagationResult: - """Map changed/unknown source freshness roots to downstream model names.""" - - changed_source_model_names: dict[SourceFreshnessIdentity, frozenset[str]] = {} - unknown_source_model_names: dict[str, frozenset[str]] = {} - error_source_model_names: dict[SourceFreshnessIdentity, frozenset[str]] = {} - stale_model_names: set[str] = set() - blocked_model_names: set[str] = set() - - identity: SourceFreshnessIdentity - for identity in source_freshness.changed_identities: - model_names: frozenset[str] = _downstream_model_names( - source_name=identity.source_name, - scope=scope, - ) - changed_source_model_names[identity] = model_names - stale_model_names.update(model_names) - - source_name: str - for source_name in source_freshness.unknown_source_names: - model_names = _downstream_model_names(source_name=source_name, scope=scope) - unknown_source_model_names[source_name] = model_names - stale_model_names.update(model_names) - - for identity, age_status in source_freshness.age_statuses.items(): - if age_status != SourceFreshnessAgeStatus.ERROR: - continue - model_names = _downstream_model_names(source_name=identity.source_name, scope=scope) - error_source_model_names[identity] = model_names - blocked_model_names.update(model_names) - - return DirectSourceFreshnessPropagationResult( - changed_source_model_names=changed_source_model_names, - unknown_source_model_names=unknown_source_model_names, - error_source_model_names=error_source_model_names, - stale_model_names=frozenset(stale_model_names), - blocked_model_names=frozenset(blocked_model_names), - ) - - -def _downstream_model_names(*, source_name: str, scope: PlannerScope) -> frozenset[str]: - source_key: CompiledObjectKey = CompiledObjectKey( - resource_type=CompiledResourceType.SOURCE, - name=source_name, - ) - return build_downstream_model_name_closure( - start_keys=(source_key,), - downstream_deps=scope.downstream_deps, - ) diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/sql.py b/src/sqlbuild/compiler/source_freshness/_helpers/sql.py deleted file mode 100644 index e907ae02ca..0000000000 --- a/src/sqlbuild/compiler/source_freshness/_helpers/sql.py +++ /dev/null @@ -1,119 +0,0 @@ -"""SQL generation helpers for direct source freshness storage.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( - render_state_table_create_sql, -) -from sqlbuild.compiler.source_freshness.constants import ( - COLUMN_DATA_VERSION, - COLUMN_DATA_VERSION_HASH, - COLUMN_OBSERVED_AT, - COLUMN_RUN_ID, - COLUMN_SOURCE_NAME, - COLUMN_STRATEGY, - COLUMN_TARGET_DATABASE, - COLUMN_TARGET_NAME, - COLUMN_TARGET_SCHEMA, - COLUMN_VALUE_KIND, - SOURCE_FRESHNESS_COLUMN_TYPES, - SOURCE_FRESHNESS_COLUMNS, - SOURCE_FRESHNESS_TABLE_NAME, -) -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessInputError - -_REQUIRED_SOURCE_FRESHNESS_COLUMNS: frozenset[str] = frozenset( - { - COLUMN_SOURCE_NAME, - COLUMN_RUN_ID, - COLUMN_STRATEGY, - COLUMN_VALUE_KIND, - COLUMN_DATA_VERSION_HASH, - COLUMN_OBSERVED_AT, - } -) - - -def build_qualified_table_name( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], -) -> str: - """Build the fully qualified source freshness table name for a target schema.""" - - qualified_name: str | None = render_qualified_name( - database=database, - schema=schema, - name=SOURCE_FRESHNESS_TABLE_NAME, - ) - if qualified_name is None: - raise SourceFreshnessInputError("source freshness table requires a target schema") - return qualified_name - - -def build_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - """Build a CREATE TABLE IF NOT EXISTS statement for source freshness state.""" - - return render_state_table_create_sql( - qualified_name=build_qualified_table_name( - database=database, schema=schema, render_qualified_name=render_qualified_name - ), - columns=SOURCE_FRESHNESS_COLUMNS, - column_types=SOURCE_FRESHNESS_COLUMN_TYPES, - required_columns=_REQUIRED_SOURCE_FRESHNESS_COLUMNS, - render_framework_type=render_framework_type, - ) - - -def build_read_latest_sql( - *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] -) -> str: - """Build a windowed SELECT for the latest source freshness row per identity.""" - - qualified_name: str = build_qualified_table_name( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - ) - selected_columns: str = _source_freshness_select_columns() - return ( - f"SELECT {selected_columns} " - f"FROM (" - f"SELECT {selected_columns}, " - f"ROW_NUMBER() OVER (" - f"PARTITION BY " - f"{COLUMN_SOURCE_NAME}, " - f"{COLUMN_TARGET_DATABASE}, " - f"{COLUMN_TARGET_SCHEMA}, " - f"{COLUMN_TARGET_NAME} " - f"ORDER BY {COLUMN_OBSERVED_AT} DESC, {COLUMN_RUN_ID} DESC" - f") AS __sqlbuild_latest_rank " - f"FROM {qualified_name}" - f") AS __sqlbuild_latest_source_freshness " - f"WHERE __sqlbuild_latest_rank = 1" - ) - - -def _source_freshness_select_columns() -> str: - return ( - f"{COLUMN_SOURCE_NAME}, " - f"{COLUMN_TARGET_DATABASE}, " - f"{COLUMN_TARGET_SCHEMA}, " - f"{COLUMN_TARGET_NAME}, " - f"{COLUMN_RUN_ID}, " - f"{COLUMN_STRATEGY}, " - f"{COLUMN_VALUE_KIND}, " - f"{COLUMN_DATA_VERSION}, " - f"{COLUMN_DATA_VERSION_HASH}, " - f"{COLUMN_OBSERVED_AT}" - ) diff --git a/src/sqlbuild/compiler/source_freshness/constants.py b/src/sqlbuild/compiler/source_freshness/constants.py index dda3206687..0cd633a952 100644 --- a/src/sqlbuild/compiler/source_freshness/constants.py +++ b/src/sqlbuild/compiler/source_freshness/constants.py @@ -5,8 +5,6 @@ from sqlbuild.sql_values.types import StateSqlValueType SOURCE_FRESHNESS_TABLE_NAME: str = "_sqlbuild_source_freshness" -PHYSICAL_TABLE_SOURCE_ERROR_FRAGMENT: str = "requires a physical table source" -INCOMPLETE_CONFIGURATION_ERROR_FRAGMENT: str = "incomplete" COLUMN_SOURCE_NAME: str = "source_name" COLUMN_TARGET_DATABASE: str = "target_database" diff --git a/src/sqlbuild/compiler/source_freshness/exceptions.py b/src/sqlbuild/compiler/source_freshness/exceptions.py deleted file mode 100644 index bb93a51fd9..0000000000 --- a/src/sqlbuild/compiler/source_freshness/exceptions.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Direct source freshness state exceptions.""" - -from __future__ import annotations - - -class SourceFreshnessInputError(ValueError): - """Raised when persisted source freshness state cannot be read safely.""" - - -class SourceFreshnessObservationError(ValueError): - """Raised when source freshness cannot be observed safely.""" diff --git a/src/sqlbuild/compiler/source_freshness/main/__init__.py b/src/sqlbuild/compiler/source_freshness/main/__init__.py deleted file mode 100644 index 103f169dd0..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Direct source freshness state operations.""" diff --git a/src/sqlbuild/compiler/source_freshness/main/_planning.py b/src/sqlbuild/compiler/source_freshness/main/_planning.py deleted file mode 100644 index 21e835d36b..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/_planning.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Public direct source freshness planning entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable -from datetime import datetime -from typing import Any - -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.compiler.source_freshness._helpers.planning import ( - build_direct_source_freshness_planning_result as _build_planning_result, -) -from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult -from sqlbuild.spec.contracts.models import SourceEntry - - -def build_direct_source_freshness_planning_result( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - state_database: str | None, - state_schemas: tuple[str, ...], - observed_at: datetime, - run_id: str, - render_qualified_name: Callable[..., str | None], - state_table_exists_by_schema: dict[str, bool], - filter_previous_to_sources: bool = False, -) -> DirectSourceFreshnessPlanningResult: - """Observe direct source freshness and compare it to latest append-only state.""" - - return _build_planning_result( - adapter=adapter, - connection=connection, - sources=sources, - state_database=state_database, - state_schemas=state_schemas, - observed_at=observed_at, - run_id=run_id, - render_qualified_name=render_qualified_name, - state_table_exists_by_schema=state_table_exists_by_schema, - filter_previous_to_sources=filter_previous_to_sources, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/_propagation.py b/src/sqlbuild/compiler/source_freshness/main/_propagation.py deleted file mode 100644 index 8a12702483..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/_propagation.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Public direct source freshness downstream propagation entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.models import PlannerScope -from sqlbuild.compiler.source_freshness._helpers import propagation as propagation_helpers -from sqlbuild.compiler.source_freshness.models import ( - DirectSourceFreshnessPlanningResult, - DirectSourceFreshnessPropagationResult, -) - - -def build_direct_source_freshness_propagation_result( - *, - source_freshness: DirectSourceFreshnessPlanningResult, - scope: PlannerScope, -) -> DirectSourceFreshnessPropagationResult: - """Map changed/unknown source freshness roots to downstream model names.""" - - return propagation_helpers.build_direct_source_freshness_propagation_result( - source_freshness=source_freshness, - scope=scope, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/adapter_observation.py b/src/sqlbuild/compiler/source_freshness/main/adapter_observation.py deleted file mode 100644 index 91263cf282..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/adapter_observation.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Public adapter metadata source freshness batch observation entrypoint.""" - -from __future__ import annotations - -from datetime import datetime -from typing import Any - -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.compiler.source_freshness._helpers.observation import ( - observe_adapter_sources_freshness as _observe_adapter_sources_freshness, -) -from sqlbuild.compiler.source_freshness.models import AdapterSourceFreshnessBatch -from sqlbuild.spec.contracts.models import SourceEntry - - -def observe_adapter_sources_freshness( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - observed_at: datetime, -) -> AdapterSourceFreshnessBatch: - """Observe adapter metadata freshness per source, batching the warehouse lookups.""" - - return _observe_adapter_sources_freshness( - adapter=adapter, - connection=connection, - sources=sources, - observed_at=observed_at, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/create_table_sql.py b/src/sqlbuild/compiler/source_freshness/main/create_table_sql.py deleted file mode 100644 index cd711ea0b0..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/create_table_sql.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Public direct source freshness table DDL rendering entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.compiler.source_freshness._helpers.sql import ( - build_create_table_sql as _build_create_table_sql, -) - - -def build_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - """Build source freshness table DDL through the public entrypoint.""" - - return _build_create_table_sql( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - render_framework_type=render_framework_type, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/observation.py b/src/sqlbuild/compiler/source_freshness/main/observation.py deleted file mode 100644 index 3ac5fa2a24..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/observation.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Public shared source freshness observation entrypoint.""" - -from __future__ import annotations - -from datetime import datetime -from typing import Any - -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.compiler.source_freshness._helpers.observation import ( - observe_configured_source_freshness as _observe_configured_source_freshness, -) -from sqlbuild.compiler.source_freshness.models import SourceFreshnessObservation -from sqlbuild.spec.contracts.models import SourceEntry - - -def observe_configured_source_freshness( - *, - adapter: StrictAdapter, - connection: Any, - source: SourceEntry, - observed_at: datetime, -) -> SourceFreshnessObservation: - """Observe one source freshness config and return a comparable data version.""" - - return _observe_configured_source_freshness( - adapter=adapter, - connection=connection, - source=source, - observed_at=observed_at, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/read.py b/src/sqlbuild/compiler/source_freshness/main/read.py deleted file mode 100644 index 58f9a8fd17..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/read.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Direct source freshness read operations.""" - -from __future__ import annotations - -from collections.abc import Callable -from datetime import datetime -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.source_freshness._helpers.datetime import normalize_presumed_utc_datetime -from sqlbuild.compiler.source_freshness._helpers.sql import ( - build_qualified_table_name, -) -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessInputError -from sqlbuild.compiler.source_freshness.models import ( - SourceFreshnessIdentity, - SourceFreshnessRecord, - SourceFreshnessSet, -) - - -def read_latest_source_freshness( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - table_exists: bool, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_read_latest_sql: Callable[..., str], - source_names: tuple[str, ...] | None = None, -) -> SourceFreshnessSet: - """Read latest source freshness rows, trusting the caller-resolved table existence.""" - - qualified_name: str = build_qualified_table_name( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - ) - if not table_exists: - return SourceFreshnessSet(schema=schema, records={}) - - read_sql: str = render_read_latest_sql( - database=database, - schema=schema, - ) - if source_names is not None: - if not source_names: - return SourceFreshnessSet(schema=schema, records={}) - literals: str = ", ".join( - "'" + source_name.replace("'", "''") + "'" for source_name in source_names - ) - read_sql = ( - f"SELECT * FROM ({read_sql}) AS __sqlbuild_relevant WHERE source_name IN ({literals})" - ) - try: - result: Any = execute(connection=connection, sql=read_sql) - except Exception as error: - raise SourceFreshnessInputError( - f"Unable to read source freshness records from {qualified_name}. This can happen " - "after upgrading from an older sqlbuild version; delete or rebuild the SQLBuild " - "source freshness table to regenerate source freshness state." - ) from error - rows: list[tuple[Any, ...]] = result.fetchall() - records: dict[SourceFreshnessIdentity, SourceFreshnessRecord] = {} - row: tuple[Any, ...] - for row in rows: - record: SourceFreshnessRecord = _row_to_source_freshness_record(row) - records[record.identity] = record - return SourceFreshnessSet(schema=schema, records=records) - - -def _row_to_source_freshness_record(row: tuple[Any, ...]) -> SourceFreshnessRecord: - raw_target_database: Any = row[1] - raw_target_schema: Any = row[2] - raw_target_name: Any = row[3] - raw_data_version: Any = row[7] - raw_observed_at: Any = row[9] - observed_at: datetime = ( - raw_observed_at - if isinstance(raw_observed_at, datetime) - else datetime.fromisoformat(str(raw_observed_at)) - ) - observed_at = normalize_presumed_utc_datetime(value=observed_at) - return SourceFreshnessRecord( - source_name=str(row[0]), - target_database=str(raw_target_database) if raw_target_database is not None else None, - target_schema=str(raw_target_schema) if raw_target_schema is not None else None, - target_name=str(raw_target_name) if raw_target_name is not None else None, - run_id=str(row[4]), - strategy=str(row[5]), - value_kind=str(row[6]), - data_version=str(raw_data_version) if raw_data_version is not None else None, - data_version_hash=str(row[8]), - observed_at=observed_at, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/read_latest_sql.py b/src/sqlbuild/compiler/source_freshness/main/read_latest_sql.py deleted file mode 100644 index 7677518077..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/read_latest_sql.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Public direct source freshness latest-read SQL rendering entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.compiler.source_freshness._helpers.sql import ( - build_read_latest_sql as _build_read_latest_sql, -) - - -def build_read_latest_sql( - *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] -) -> str: - """Build source freshness latest-read SQL through the public entrypoint.""" - - return _build_read_latest_sql( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/write.py b/src/sqlbuild/compiler/source_freshness/main/write.py deleted file mode 100644 index 6014cdabf2..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/write.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Direct source freshness write operations.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.source_freshness._helpers.sql import ( - build_create_table_sql, -) -from sqlbuild.compiler.source_freshness.models import ( - SourceFreshnessRecord, - SourceFreshnessRenderers, -) - - -def write_source_freshness_records( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - database: str | None, - schema: str, - records: tuple[SourceFreshnessRecord, ...], - renderers: SourceFreshnessRenderers, -) -> None: - """Append source freshness rows, creating the table once if needed.""" - - if not records: - return - create_sql: str = ( - renderers.render_create_table_sql(database=database, schema=schema) - if renderers.render_create_table_sql is not None - else build_create_table_sql( - database=database, - schema=schema, - render_qualified_name=renderers.render_qualified_name, - render_framework_type=renderers.render_framework_type, - ) - ) - _ = execute(connection=connection, sql=create_sql) - if renderers.render_create_index_sqls is not None: - index_sql: str - for index_sql in renderers.render_create_index_sqls(database=database, schema=schema): - _ = execute(connection=connection, sql=index_sql) - _ = execute( - connection=connection, - sql=renderers.render_insert_records_sql( - database=database, - schema=schema, - records=records, - ), - ) diff --git a/src/sqlbuild/compiler/source_freshness/models.py b/src/sqlbuild/compiler/source_freshness/models.py index 507b047ad1..b12173a8a1 100644 --- a/src/sqlbuild/compiler/source_freshness/models.py +++ b/src/sqlbuild/compiler/source_freshness/models.py @@ -6,8 +6,8 @@ from dataclasses import dataclass, field from datetime import datetime +from sqlbuild._native import require_aware_utc_datetime from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.compiler.source_freshness._helpers.datetime import require_aware_utc_datetime from sqlbuild.compiler.source_freshness.types import ( SourceFreshnessAgeStatus, SourceFreshnessUnknownReason, diff --git a/src/sqlbuild/compiler/sql_analysis/main/_normalize_analysis.py b/src/sqlbuild/compiler/sql_analysis/main/_normalize_analysis.py deleted file mode 100644 index d21014a46c..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_normalize_analysis.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Normalize analysis SQL using native offset-preserving transformations.""" - -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.sql_analysis.types import NativePositionsModule - - -def normalize_analysis_sql( - *, - sql: str, - dialect: str | None, - stubs: dict[str, str] | None = None, - placeholders: dict[str, str] | None = None, -) -> str: - return cast(NativePositionsModule, _native).normalize_analysis_sql( - { - "sql": sql, - "dialect": dialect or "generic", - "stubs": stubs or {}, - "placeholders": placeholders or {}, - } - ) diff --git a/src/sqlbuild/cost/_helpers/allocation.py b/src/sqlbuild/cost/_helpers/allocation.py deleted file mode 100644 index 8729a369df..0000000000 --- a/src/sqlbuild/cost/_helpers/allocation.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Concurrency-safe Snowflake busy-compute attribution.""" - -from __future__ import annotations - -from collections import defaultdict -from datetime import datetime -from decimal import Decimal - -from sqlbuild.cost.constants import STANDARD_WAREHOUSE_TYPE -from sqlbuild.cost.models import QueryCostObservation, ResourceCost, RunCostSummary -from sqlbuild.cost.types import CostStatus - -_CREDITS_PER_HOUR: dict[str, Decimal] = { - "X-SMALL": Decimal(1), - "SMALL": Decimal(2), - "MEDIUM": Decimal(4), - "LARGE": Decimal(8), - "X-LARGE": Decimal(16), - "2X-LARGE": Decimal(32), - "3X-LARGE": Decimal(64), - "4X-LARGE": Decimal(128), - "5X-LARGE": Decimal(256), - "6X-LARGE": Decimal(512), -} - - -def allocate_run_cost( - *, - observations: tuple[QueryCostObservation, ...], - run_id: str, - usd_per_credit: Decimal, - rate_source: str, - result_limit_reached: bool = False, -) -> RunCostSummary: - run_queries: tuple[QueryCostObservation, ...] = tuple( - observation for observation in observations if observation.run_id == run_id - ) - if not run_queries: - return RunCostSummary( - status=CostStatus.PENDING, - usd_per_credit=usd_per_credit, - rate_source=rate_source, - message="Snowflake query history is not complete yet.", - ) - - attributed: dict[str, Decimal] = defaultdict(Decimal) - by_warehouse: dict[tuple[str, str, str | None, int | None], list[QueryCostObservation]] = ( - defaultdict(list) - ) - for observation in observations: - by_warehouse[ - ( - observation.warehouse_name, - observation.warehouse_size, - observation.warehouse_type, - observation.cluster_number, - ) - ].append(observation) - - for warehouse_queries in by_warehouse.values(): - boundaries: list[datetime] = _query_boundaries(queries=warehouse_queries) - for start, end in zip(boundaries, boundaries[1:], strict=False): - if end <= start: - continue - active: tuple[QueryCostObservation, ...] = tuple( - query - for query in warehouse_queries - if query.started_at < end and query.completed_at > start - ) - if not active: - continue - seconds: Decimal = Decimal(str((end - start).total_seconds())) / Decimal(len(active)) - for query in active: - if query.run_id == run_id: - attributed[query.query_id] += seconds - - grouped: dict[tuple[str, str, str, str, str | None, int | None], list[QueryCostObservation]] = ( - defaultdict(list) - ) - for query in run_queries: - grouped[ - ( - query.resource_type or "unknown", - query.resource_name or "unknown", - query.warehouse_name, - query.warehouse_size, - query.warehouse_type, - query.cluster_number, - ) - ].append(query) - - limitations: list[str] = [] - resources: list[ResourceCost] = [] - for ( - resource_type, - resource_name, - warehouse_name, - warehouse_size, - warehouse_type, - cluster_number, - ), queries in grouped.items(): - rate: Decimal | None = _CREDITS_PER_HOUR.get(warehouse_size.upper()) - if warehouse_type is None: - limitations.append(f"Unknown warehouse type for {warehouse_name}") - elif warehouse_type.upper() != STANDARD_WAREHOUSE_TYPE: - limitations.append(f"Unsupported Snowflake warehouse type: {warehouse_type}") - continue - if cluster_number is None: - limitations.append(f"Unknown cluster number for {warehouse_name}") - if rate is None: - limitations.append(f"Unknown Snowflake warehouse size: {warehouse_size}") - continue - seconds = sum((attributed[query.query_id] for query in queries), Decimal(0)) - credits: Decimal = seconds * rate / Decimal(3600) - resources.append( - ResourceCost( - resource_type=resource_type, - resource_name=resource_name, - warehouse_name=warehouse_name, - warehouse_size=warehouse_size, - warehouse_type=warehouse_type, - cluster_number=cluster_number, - query_count=len(queries), - attributed_seconds=seconds, - bytes_scanned=sum(query.bytes_scanned for query in queries), - estimated_compute_credits=credits, - estimated_usd=credits * usd_per_credit, - ) - ) - if result_limit_reached: - limitations.append("Snowflake query-history result limit was reached.") - resources.sort( - key=lambda item: ( - -item.estimated_usd, - item.resource_name, - item.resource_type, - item.warehouse_name, - item.warehouse_size, - item.warehouse_type or "", - -1 if item.cluster_number is None else item.cluster_number, - ) - ) - return RunCostSummary( - status=CostStatus.PARTIAL if limitations else CostStatus.COMPLETE, - usd_per_credit=usd_per_credit, - rate_source=rate_source, - resources=tuple(resources), - query_count=len(run_queries), - attributed_seconds=sum((item.attributed_seconds for item in resources), Decimal(0)), - bytes_scanned=sum(item.bytes_scanned for item in resources), - estimated_compute_credits=sum( - (item.estimated_compute_credits for item in resources), Decimal(0) - ), - estimated_usd=sum((item.estimated_usd for item in resources), Decimal(0)), - limitations=tuple(sorted(set(limitations))), - ) - - -def _query_boundaries(*, queries: list[QueryCostObservation]) -> list[datetime]: - boundaries: set[datetime] = set() - for query in queries: - boundaries.add(query.started_at) - boundaries.add(query.completed_at) - return sorted(boundaries) diff --git a/src/sqlbuild/cost/_helpers/context.py b/src/sqlbuild/cost/_helpers/context.py deleted file mode 100644 index a3db6f4c2f..0000000000 --- a/src/sqlbuild/cost/_helpers/context.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Per-resource cost attribution context.""" - -from __future__ import annotations - -from collections.abc import Callable, Iterator -from contextlib import contextmanager -from contextvars import ContextVar, Token -from pathlib import Path - -from sqlbuild.cost.models import CostResourceContext, StatementExecutionTelemetry - -_CURRENT: ContextVar[CostResourceContext | None] = ContextVar( - "sqlbuild_cost_resource_context", default=None -) - - -def current_cost_resource_context() -> CostResourceContext | None: - return _CURRENT.get() - - -@contextmanager -def cost_resource_context( - *, - run_id: str, - resource_type: str, - resource_name: str, - ledger_path: Path | None = None, - phase: str = "execute", - attempt: int = 1, - on_statement_complete: Callable[[StatementExecutionTelemetry], None] | None = None, -) -> Iterator[None]: - token: Token[CostResourceContext | None] = _CURRENT.set( - CostResourceContext( - run_id=run_id, - resource_type=resource_type, - resource_name=resource_name, - ledger_path=ledger_path, - phase=phase, - attempt=attempt, - on_statement_complete=on_statement_complete, - ) - ) - try: - yield - finally: - _CURRENT.reset(token) diff --git a/src/sqlbuild/cost/_helpers/ledger.py b/src/sqlbuild/cost/_helpers/ledger.py deleted file mode 100644 index ef75a02edb..0000000000 --- a/src/sqlbuild/cost/_helpers/ledger.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Best-effort append-only statement telemetry persistence.""" - -from __future__ import annotations - -import hashlib -import json -import threading -from datetime import datetime -from pathlib import Path -from typing import Any - -from sqlbuild.cost.constants import COST_TELEMETRY_HEALTH -from sqlbuild.cost.exceptions import CostArtifactError -from sqlbuild.cost.models import CostResourceContext, StatementLedgerEntry - -_LOCK: threading.Lock = threading.Lock() - - -def record_statement( - *, - context: CostResourceContext, - statement_id: str, - sql: str, - query_id: object, - status: str, - started_at: datetime, - completed_at: datetime, - error: Exception | None = None, -) -> None: - """Append statement identity without allowing telemetry I/O to affect execution.""" - - if context.ledger_path is None: - return - try: - _append_statement( - path=context.ledger_path, - payload={ - "run_id": context.run_id, - "statement_id": statement_id, - "resource_type": context.resource_type, - "resource_name": context.resource_name, - "phase": context.phase, - "attempt": context.attempt, - "query_id": query_id if isinstance(query_id, str) and query_id else None, - "status": status, - "started_at": started_at.isoformat(), - "completed_at": completed_at.isoformat(), - "sql_sha256": hashlib.sha256(sql.encode("utf-8")).hexdigest(), - "error_type": None if error is None else type(error).__name__, - }, - ) - except Exception as error: - COST_TELEMETRY_HEALTH.mark_ledger_failure(run_id=context.run_id, error=error) - return - - -def read_statement_ledger(*, path: Path, run_id: str) -> tuple[StatementLedgerEntry, ...]: - """Read a stable snapshot of valid statement ledger entries.""" - - if not path.is_file(): - return () - entries: list[StatementLedgerEntry] = [] - with _LOCK: - try: - lines: list[str] = path.read_text(encoding="utf-8").splitlines() - except OSError as error: - COST_TELEMETRY_HEALTH.mark_ledger_failure(run_id=run_id, error=error) - return () - for line in lines: - try: - payload: dict[str, Any] = json.loads(line) - query_id_value: object = payload.get("query_id") - entries.append( - StatementLedgerEntry( - statement_id=str(payload["statement_id"]), - run_id=str(payload["run_id"]), - resource_type=str(payload["resource_type"]), - resource_name=str(payload["resource_name"]), - phase=str(payload["phase"]), - attempt=int(payload["attempt"]), - query_id=( - query_id_value - if isinstance(query_id_value, str) and query_id_value - else None - ), - status=str(payload["status"]), - started_at=datetime.fromisoformat(str(payload["started_at"])), - completed_at=datetime.fromisoformat(str(payload["completed_at"])), - ) - ) - except (KeyError, TypeError, ValueError, json.JSONDecodeError) as error: - COST_TELEMETRY_HEALTH.mark_ledger_failure( - run_id=run_id, - error=CostArtifactError(f"invalid statement ledger entry ({type(error).__name__})"), - ) - continue - return tuple(entries) - - -def _append_statement(*, path: Path, payload: dict[str, object]) -> None: - with _LOCK: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("a", encoding="utf-8") as file: - file.write(json.dumps(payload, separators=(",", ":"), sort_keys=True) + "\n") - file.flush() diff --git a/src/sqlbuild/cost/_helpers/snowflake.py b/src/sqlbuild/cost/_helpers/snowflake.py deleted file mode 100644 index a04833cc78..0000000000 --- a/src/sqlbuild/cost/_helpers/snowflake.py +++ /dev/null @@ -1,625 +0,0 @@ -"""Snowflake query-history cost collection.""" - -from __future__ import annotations - -import json -import time -from dataclasses import replace -from datetime import UTC, datetime, timedelta -from decimal import Decimal -from pathlib import Path -from typing import Any - -from polyglot_sql import CreateTable, CreateView, ParseError, parse_one - -from sqlbuild.cost._helpers.allocation import allocate_run_cost -from sqlbuild.cost._helpers.ledger import read_statement_ledger -from sqlbuild.cost.constants import ( - COST_TELEMETRY_HEALTH, - INCOMPLETE_HISTORY_REASON, - LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX, - MISSING_WAREHOUSE_METADATA_REASON, - NO_WAREHOUSE_COMPUTE_REASON, - RUNNING_EXECUTION_STATUS, - SQLBUILD_QUERY_TAG_APP, -) -from sqlbuild.cost.models import ( - QueryCostObservation, - RunCostSummary, - StatementLedgerEntry, -) -from sqlbuild.cost.types import CostStatus - -_RESULT_LIMIT: int = 10_000 -_MINIMUM_SPLIT_WINDOW: timedelta = timedelta(milliseconds=1) -_COLLECTION_DEADLINE_SECONDS: float = 10.0 -_QUERY_TIMEOUT_SECONDS: str = "5" -_QUERY_HISTORY_RETENTION: timedelta = timedelta(days=7) -_MAXIMUM_CLOCK_SKEW: timedelta = timedelta(minutes=10) -_USE_QUERY_TYPE: str = "USE" -_DROP_QUERY_TYPE: str = "DROP" -_CREATE_QUERY_TYPE: str = "CREATE" -_DYNAMIC_TABLE_MODIFIER: str = "DYNAMIC" -_HOOK_RESOURCE_TYPE: str = "hook" -_VISIBLE_QUERY_LIMITATION: str = ( - "Concurrency sharing includes only queries visible to the executing Snowflake role." -) -_INCOMPLETE_OBSERVATION_REASONS: frozenset[str] = frozenset( - {MISSING_WAREHOUSE_METADATA_REASON, INCOMPLETE_HISTORY_REASON} -) - - -def collect_snowflake_run_cost( - *, - connection: Any, - database: str | None = None, - run_id: str, - started_at: datetime, - completed_at: datetime, - statement_ledger_path: Path, - usd_per_credit: Decimal, - rate_source: str, - attempts: int = 3, - retry_delay_seconds: float = 1.0, -) -> RunCostSummary: - """Collect visible history and attribute exact ledger query IDs to one run.""" - - deadline: float = time.monotonic() + _COLLECTION_DEADLINE_SECONDS - ledger: tuple[StatementLedgerEntry, ...] = read_statement_ledger( - path=statement_ledger_path, - run_id=run_id, - ) - ledger_failure: str | None = COST_TELEMETRY_HEALTH.consume_ledger_failure(run_id=run_id) - ledger_by_query_id: dict[str, StatementLedgerEntry] = { - entry.query_id: entry for entry in ledger if entry.query_id is not None - } - expected_query_ids: frozenset[str] = frozenset(ledger_by_query_id) - classification_query_ids: frozenset[str] = frozenset( - query_id - for query_id, entry in ledger_by_query_id.items() - if entry.resource_type != _HOOK_RESOURCE_TYPE - ) - if ledger and not expected_query_ids: - no_query_id_summary: RunCostSummary = _apply_ledger_failure( - summary=_partial_summary( - usd_per_credit=usd_per_credit, - rate_source=rate_source, - message="No Snowflake query IDs were available in the statement ledger.", - ), - ledger_failure=ledger_failure, - ) - return replace( - no_query_id_summary, - expected_statement_count=len(ledger), - missing_statement_count=len(ledger), - ) - - summary: RunCostSummary | None = None - observed_query_ids: frozenset[str] = frozenset() - skipped_reasons: dict[str, str] = {} - saturated = False - for attempt in range(attempts): - observations, observed_query_ids, skipped_reasons, saturated = _query_observations( - connection=connection, - database=database, - started_at=started_at, - completed_at=completed_at, - expected_query_ids=expected_query_ids, - classification_query_ids=classification_query_ids, - deadline=deadline, - ) - attributed_observations: tuple[QueryCostObservation, ...] = tuple( - _attribute_from_ledger(observation=observation, ledger=ledger_by_query_id) - for observation in observations - ) - summary = allocate_run_cost( - observations=attributed_observations, - run_id=run_id, - usd_per_credit=usd_per_credit, - rate_source=rate_source, - result_limit_reached=saturated, - ) - summary = replace( - summary, - limitations=tuple(sorted({*summary.limitations, _VISIBLE_QUERY_LIMITATION})), - ) - if expected_query_ids.issubset(observed_query_ids) and not _has_incomplete_observation( - expected_query_ids=expected_query_ids, - skipped_reasons=skipped_reasons, - ): - summary = _apply_observation_gaps( - summary=summary, - expected_statement_count=len(ledger), - expected_query_ids=expected_query_ids, - observed_query_ids=observed_query_ids, - skipped_reasons=skipped_reasons, - ) - if summary.status == CostStatus.PENDING: - summary = replace( - summary, - status=CostStatus.COMPLETE, - message="No cost-bearing Snowflake warehouse queries were observed.", - ) - return _apply_ledger_failure(summary=summary, ledger_failure=ledger_failure) - if attempt + 1 < attempts: - remaining_seconds: float = deadline - time.monotonic() - if remaining_seconds <= retry_delay_seconds: - break - time.sleep(retry_delay_seconds) - - if summary is None: - summary = _partial_summary( - usd_per_credit=usd_per_credit, - rate_source=rate_source, - message="Snowflake query history could not be collected.", - ) - missing_count: int = len(expected_query_ids.difference(observed_query_ids)) - limitations: set[str] = set(summary.limitations) - if missing_count: - limitations.add( - f"{missing_count} statement ledger query ID(s) were not visible in query history." - ) - if saturated: - limitations.add("Snowflake query-history result limit was reached.") - visible_expected_query_ids: frozenset[str] = expected_query_ids.intersection(observed_query_ids) - history_expired: bool = datetime.now(UTC) - completed_at >= _QUERY_HISTORY_RETENTION - final_status: CostStatus - if history_expired and not visible_expected_query_ids: - final_status = CostStatus.UNAVAILABLE - message = "Snowflake query history is no longer available for this run." - elif not visible_expected_query_ids and not saturated: - final_status = CostStatus.PENDING - message = "Snowflake query history is not complete yet. Run `sqb cost latest` to refresh." - else: - final_status = CostStatus.PARTIAL - message = summary.message or "Snowflake query history was only partially available." - summary = _apply_observation_gaps( - summary=replace( - summary, - status=final_status, - limitations=tuple(sorted(limitations)), - message=message, - ), - expected_statement_count=len(ledger), - expected_query_ids=expected_query_ids, - observed_query_ids=observed_query_ids, - skipped_reasons=skipped_reasons, - ) - return _apply_ledger_failure( - summary=summary, - ledger_failure=ledger_failure, - ) - - -def _query_observations( - *, - connection: Any, - database: str | None = None, - started_at: datetime, - completed_at: datetime, - expected_query_ids: frozenset[str], - classification_query_ids: frozenset[str], - deadline: float, -) -> tuple[ - tuple[QueryCostObservation, ...], - frozenset[str], - dict[str, str], - bool, -]: - open_rows: tuple[tuple[Any, ...], ...] = _query_open_history_rows( - connection=connection, - database=database, - started_at=started_at - _MAXIMUM_CLOCK_SKEW, - classification_query_ids=classification_query_ids, - deadline=deadline, - ) - saturated: bool = len(open_rows) >= _RESULT_LIMIT - if saturated: - bounded_rows, bounded_saturated = _query_history_rows( - connection=connection, - database=database, - started_at=started_at - _MAXIMUM_CLOCK_SKEW, - completed_at=completed_at + _MAXIMUM_CLOCK_SKEW, - classification_query_ids=classification_query_ids, - deadline=deadline, - ) - rows_by_query_id: dict[str, tuple[Any, ...]] = { - str(row[0]): row for row in open_rows if _is_running_history_row(row=row) - } - rows_by_query_id.update({str(row[0]): row for row in bounded_rows}) - rows: tuple[tuple[Any, ...], ...] = tuple( - rows_by_query_id[key] for key in sorted(rows_by_query_id) - ) - saturated = bounded_saturated - else: - rows = open_rows - observations: list[QueryCostObservation] = [] - observed_query_ids: set[str] = set() - skipped_reasons: dict[str, str] = {} - for row in rows: - query_id: str = str(row[0]) - observed_query_ids.add(query_id) - observation, skip_reason = _row_to_observation( - row=row, - require_complete=query_id in expected_query_ids, - ) - if observation is not None: - observations.append(observation) - elif skip_reason is not None: - skipped_reasons[query_id] = skip_reason - observations.sort(key=lambda item: (item.started_at, item.query_id)) - return tuple(observations), frozenset(observed_query_ids), skipped_reasons, saturated - - -def _is_running_history_row(*, row: tuple[Any, ...]) -> bool: - return row[7] is None or str(row[10]).upper() == RUNNING_EXECUTION_STATUS - - -def _query_open_history_rows( - *, - connection: Any, - database: str | None = None, - started_at: datetime, - classification_query_ids: frozenset[str], - deadline: float, -) -> tuple[tuple[Any, ...], ...]: - if time.monotonic() >= deadline: - raise TimeoutError("Snowflake cost collection deadline exceeded") - cursor: Any = connection.execute( - _render_open_query_history_sql( - database=database, - started_at=started_at, - classification_query_ids=classification_query_ids, - ), - statement_params={"STATEMENT_TIMEOUT_IN_SECONDS": _QUERY_TIMEOUT_SECONDS}, - ) - if time.monotonic() >= deadline: - raise TimeoutError("Snowflake cost collection deadline exceeded") - rows: tuple[tuple[Any, ...], ...] = tuple(tuple(row) for row in cursor.fetchall()) - if time.monotonic() >= deadline: - raise TimeoutError("Snowflake cost collection deadline exceeded") - return rows - - -def _query_history_rows( - *, - connection: Any, - database: str | None = None, - started_at: datetime, - completed_at: datetime, - classification_query_ids: frozenset[str], - deadline: float, -) -> tuple[tuple[tuple[Any, ...], ...], bool]: - if time.monotonic() >= deadline: - raise TimeoutError("Snowflake cost collection deadline exceeded") - cursor: Any = connection.execute( - _render_query_history_sql( - database=database, - started_at=started_at, - completed_at=completed_at, - classification_query_ids=classification_query_ids, - ), - statement_params={"STATEMENT_TIMEOUT_IN_SECONDS": _QUERY_TIMEOUT_SECONDS}, - ) - if time.monotonic() >= deadline: - raise TimeoutError("Snowflake cost collection deadline exceeded") - rows: tuple[tuple[Any, ...], ...] = tuple(tuple(row) for row in cursor.fetchall()) - if time.monotonic() >= deadline: - raise TimeoutError("Snowflake cost collection deadline exceeded") - if len(rows) < _RESULT_LIMIT: - return rows, False - if completed_at - started_at <= _MINIMUM_SPLIT_WINDOW: - return rows, True - - midpoint: datetime = started_at + (completed_at - started_at) / 2 - left_rows, left_saturated = _query_history_rows( - connection=connection, - database=database, - started_at=started_at, - completed_at=midpoint, - classification_query_ids=classification_query_ids, - deadline=deadline, - ) - right_rows, right_saturated = _query_history_rows( - connection=connection, - database=database, - started_at=midpoint, - completed_at=completed_at, - classification_query_ids=classification_query_ids, - deadline=deadline, - ) - rows_by_query_id: dict[str, tuple[Any, ...]] = { - str(row[0]): row for row in (*left_rows, *right_rows) - } - merged_rows: tuple[tuple[Any, ...], ...] = tuple( - rows_by_query_id[key] for key in sorted(rows_by_query_id) - ) - return merged_rows, left_saturated or right_saturated - - -def _row_to_observation( - *, - row: tuple[Any, ...], - require_complete: bool, -) -> tuple[QueryCostObservation | None, str | None]: - ( - query_id, - query_tag, - warehouse_name, - warehouse_size, - warehouse_type, - cluster_number, - start_time, - end_time, - execution_ms, - bytes_scanned, - execution_status, - query_type, - query_text, - observed_at, - ) = row - milliseconds: int = max(0, int(execution_ms or 0)) - if _is_metadata_only_statement(query_type=query_type, query_text=query_text): - return None, NO_WAREHOUSE_COMPUTE_REASON - running: bool = end_time is None or str(execution_status).upper() == RUNNING_EXECUTION_STATUS - if require_complete and running: - return None, INCOMPLETE_HISTORY_REASON - if milliseconds == 0 or warehouse_name is None or warehouse_size is None: - return None, MISSING_WAREHOUSE_METADATA_REASON - tag: dict[str, object] | None = _parse_sqlbuild_tag(query_tag) - effective_end: datetime = observed_at if running else end_time - active_start: datetime = max( - start_time, - effective_end - timedelta(milliseconds=milliseconds), - ) - if effective_end <= active_start: - return None, NO_WAREHOUSE_COMPUTE_REASON - return ( - QueryCostObservation( - query_id=str(query_id), - warehouse_name=str(warehouse_name), - warehouse_size=str(warehouse_size), - warehouse_type=None if warehouse_type is None else str(warehouse_type), - cluster_number=None if cluster_number is None else int(cluster_number), - started_at=active_start, - completed_at=effective_end, - execution_ms=milliseconds, - bytes_scanned=int(bytes_scanned or 0), - execution_status=str(execution_status), - run_id=None if tag is None else _optional_string(tag.get("run_id")), - resource_type=None if tag is None else _optional_string(tag.get("resource_type")), - resource_name=None if tag is None else _optional_string(tag.get("resource_name")), - ), - None, - ) - - -def _attribute_from_ledger( - *, - observation: QueryCostObservation, - ledger: dict[str, StatementLedgerEntry], -) -> QueryCostObservation: - entry: StatementLedgerEntry | None = ledger.get(observation.query_id) - if entry is None: - return replace( - observation, - run_id=None, - resource_type=None, - resource_name=None, - ) - return replace( - observation, - run_id=entry.run_id, - resource_type=entry.resource_type, - resource_name=entry.resource_name, - ) - - -def _render_query_history_sql( - *, - database: str | None = None, - started_at: datetime, - completed_at: datetime, - classification_query_ids: frozenset[str], -) -> str: - start: str = started_at.isoformat() - end: str = completed_at.isoformat() - query_text_expression: str = _render_query_text_expression( - classification_query_ids=classification_query_ids - ) - query_history_function: str = _render_query_history_function(database=database) - return f""" -SELECT - QUERY_ID, - QUERY_TAG, - WAREHOUSE_NAME, - WAREHOUSE_SIZE, - WAREHOUSE_TYPE, - CLUSTER_NUMBER, - START_TIME, - END_TIME, - EXECUTION_TIME, - BYTES_SCANNED, - EXECUTION_STATUS, - QUERY_TYPE, - {query_text_expression}, - CURRENT_TIMESTAMP() -FROM TABLE({query_history_function}( - END_TIME_RANGE_START => TO_TIMESTAMP_LTZ('{start}'), - END_TIME_RANGE_END => TO_TIMESTAMP_LTZ('{end}'), - RESULT_LIMIT => {_RESULT_LIMIT} -)) -ORDER BY START_TIME, QUERY_ID -""".strip() - - -def _render_open_query_history_sql( - *, - database: str | None = None, - started_at: datetime, - classification_query_ids: frozenset[str], -) -> str: - start: str = started_at.isoformat() - query_text_expression: str = _render_query_text_expression( - classification_query_ids=classification_query_ids - ) - query_history_function: str = _render_query_history_function(database=database) - return f""" -SELECT - QUERY_ID, - QUERY_TAG, - WAREHOUSE_NAME, - WAREHOUSE_SIZE, - WAREHOUSE_TYPE, - CLUSTER_NUMBER, - START_TIME, - END_TIME, - EXECUTION_TIME, - BYTES_SCANNED, - EXECUTION_STATUS, - QUERY_TYPE, - {query_text_expression}, - CURRENT_TIMESTAMP() -FROM TABLE({query_history_function}( - END_TIME_RANGE_START => TO_TIMESTAMP_LTZ('{start}'), - RESULT_LIMIT => {_RESULT_LIMIT} -)) -ORDER BY START_TIME, QUERY_ID -""".strip() - - -def _render_query_history_function(*, database: str | None) -> str: - if database is None: - return "INFORMATION_SCHEMA.QUERY_HISTORY" - quoted_database: str = database.upper().replace('"', '""') - return f'"{quoted_database}".INFORMATION_SCHEMA.QUERY_HISTORY' - - -def _render_query_text_expression(*, classification_query_ids: frozenset[str]) -> str: - if not classification_query_ids: - return "NULL AS QUERY_TEXT" - query_ids: str = ", ".join( - f"'{query_id.replace(chr(39), chr(39) * 2)}'" - for query_id in sorted(classification_query_ids) - ) - return ( - f"CASE WHEN QUERY_ID IN ({query_ids}) " - "AND (QUERY_TYPE = 'CREATE' OR QUERY_TYPE LIKE 'CREATE_%') " - "THEN QUERY_TEXT END AS QUERY_TEXT" - ) - - -def _parse_sqlbuild_tag(value: object) -> dict[str, object] | None: - if not isinstance(value, str) or not value: - return None - try: - payload: object = json.loads(value) - except json.JSONDecodeError: - return None - if not isinstance(payload, dict): - return None - if payload.get("app") != SQLBUILD_QUERY_TAG_APP or payload.get("v") != 1: - return None - return payload - - -def _is_metadata_only_statement(*, query_type: object, query_text: object) -> bool: - """Identify statements Snowflake executes without warehouse compute.""" - - normalized_type: str = str(query_type or "").upper().replace(" ", "_") - if normalized_type == _USE_QUERY_TYPE or normalized_type.startswith(f"{_USE_QUERY_TYPE}_"): - return True - if normalized_type == _DROP_QUERY_TYPE or normalized_type.startswith(f"{_DROP_QUERY_TYPE}_"): - return True - if normalized_type != _CREATE_QUERY_TYPE and not normalized_type.startswith( - f"{_CREATE_QUERY_TYPE}_" - ): - return False - if not isinstance(query_text, str) or not query_text.strip(): - return False - try: - statement: Any = parse_one(query_text, dialect="snowflake") - except ParseError: - return False - if isinstance(statement, CreateTable): - return not any( - ( - statement.args.get("as_select") is not None, - statement.args.get("clone_source") is not None, - statement.args.get("using_template") is not None, - str(statement.args.get("table_modifier") or "").upper() == _DYNAMIC_TABLE_MODIFIER, - ) - ) - if isinstance(statement, CreateView) and bool(statement.args.get("materialized")): - return False - return type(statement).__name__.startswith("Create") - - -def _optional_string(value: object) -> str | None: - return value if isinstance(value, str) and value else None - - -def _partial_summary(*, usd_per_credit: Decimal, rate_source: str, message: str) -> RunCostSummary: - return RunCostSummary( - status=CostStatus.PARTIAL, - usd_per_credit=usd_per_credit, - rate_source=rate_source, - message=message, - ) - - -def _apply_ledger_failure(*, summary: RunCostSummary, ledger_failure: str | None) -> RunCostSummary: - if ledger_failure is None: - return summary - limitation: str = f"{LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX}{ledger_failure})." - return replace( - summary, - status=CostStatus.PARTIAL, - limitations=tuple(sorted({*summary.limitations, limitation})), - message=summary.message or "Cost telemetry was only partially persisted.", - ) - - -def _apply_observation_gaps( - *, - summary: RunCostSummary, - expected_statement_count: int, - expected_query_ids: frozenset[str], - observed_query_ids: frozenset[str], - skipped_reasons: dict[str, str], -) -> RunCostSummary: - relevant_skips: dict[str, str] = { - query_id: reason - for query_id, reason in skipped_reasons.items() - if query_id in expected_query_ids - } - limitations: set[str] = set(summary.limitations) - if MISSING_WAREHOUSE_METADATA_REASON in relevant_skips.values(): - limitations.add("One or more run statements had missing Snowflake warehouse metadata.") - if INCOMPLETE_HISTORY_REASON in relevant_skips.values(): - limitations.add("One or more run statements were still incomplete in query history.") - status: CostStatus = summary.status - if MISSING_WAREHOUSE_METADATA_REASON in relevant_skips.values(): - status = CostStatus.PARTIAL - elif INCOMPLETE_HISTORY_REASON in relevant_skips.values() and summary.query_count: - status = CostStatus.PARTIAL - return replace( - summary, - status=status, - limitations=tuple(sorted(limitations)), - expected_statement_count=expected_statement_count, - observed_statement_count=len(expected_query_ids.intersection(observed_query_ids)), - missing_statement_count=( - expected_statement_count - len(expected_query_ids.intersection(observed_query_ids)) - ), - skipped_statement_count=len(relevant_skips), - ) - - -def _has_incomplete_observation( - *, expected_query_ids: frozenset[str], skipped_reasons: dict[str, str] -) -> bool: - return any( - query_id in expected_query_ids and reason in _INCOMPLETE_OBSERVATION_REASONS - for query_id, reason in skipped_reasons.items() - ) diff --git a/src/sqlbuild/cost/classes/cost_context.py b/src/sqlbuild/cost/classes/cost_context.py deleted file mode 100644 index 0b769e2785..0000000000 --- a/src/sqlbuild/cost/classes/cost_context.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Run-cost execution context.""" - -from collections.abc import Callable -from contextlib import AbstractContextManager, nullcontext -from pathlib import Path - -from sqlbuild.cost._helpers.context import ( - cost_resource_context, - current_cost_resource_context, -) -from sqlbuild.cost.models import CostResourceContext, StatementExecutionTelemetry - - -class CostContext: - @staticmethod - def current() -> CostResourceContext | None: - return current_cost_resource_context() - - @staticmethod - def scope( - *, - run_id: str, - resource_type: str, - resource_name: str, - ledger_path: Path | None = None, - phase: str = "execute", - attempt: int = 1, - on_statement_complete: Callable[[StatementExecutionTelemetry], None] | None = None, - ) -> AbstractContextManager[None]: - current: CostResourceContext | None = CostContext.current() - return cost_resource_context( - run_id=run_id, - resource_type=resource_type, - resource_name=resource_name, - ledger_path=( - ledger_path - if ledger_path is not None - else (None if current is None else current.ledger_path) - ), - phase=phase, - attempt=attempt, - on_statement_complete=( - on_statement_complete - if on_statement_complete is not None - else (None if current is None else current.on_statement_complete) - ), - ) - - @staticmethod - def resource_scope( - *, - resource_type: str, - resource_name: str, - phase: str | None = None, - attempt: int = 1, - ) -> AbstractContextManager[None]: - current: CostResourceContext | None = CostContext.current() - if current is None: - return nullcontext() - return CostContext.scope( - run_id=current.run_id, - resource_type=resource_type, - resource_name=resource_name, - ledger_path=current.ledger_path, - phase=current.phase if phase is None else phase, - attempt=attempt, - on_statement_complete=current.on_statement_complete, - ) diff --git a/src/sqlbuild/cost/classes/cost_telemetry_health.py b/src/sqlbuild/cost/classes/cost_telemetry_health.py deleted file mode 100644 index 358450ed1f..0000000000 --- a/src/sqlbuild/cost/classes/cost_telemetry_health.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Process-local telemetry failure tracking for the active build.""" - -import threading - - -class CostTelemetryHealth: - def __init__(self) -> None: - self._lock: threading.Lock = threading.Lock() - self._ledger_failures: dict[str, str] = {} - - def mark_ledger_failure(self, *, run_id: str, error: Exception) -> None: - with self._lock: - self._ledger_failures.setdefault(run_id, type(error).__name__) - - def consume_ledger_failure(self, *, run_id: str) -> str | None: - with self._lock: - return self._ledger_failures.pop(run_id, None) diff --git a/src/sqlbuild/cost/classes/statement_ledger.py b/src/sqlbuild/cost/classes/statement_ledger.py deleted file mode 100644 index 057df118eb..0000000000 --- a/src/sqlbuild/cost/classes/statement_ledger.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Best-effort statement ledger.""" - -from datetime import datetime - -from sqlbuild.cost._helpers.ledger import record_statement -from sqlbuild.cost.models import CostResourceContext - - -class StatementLedger: - @staticmethod - def record( - *, - context: CostResourceContext, - statement_id: str, - sql: str, - query_id: object, - status: str, - started_at: datetime, - completed_at: datetime, - error: Exception | None = None, - ) -> None: - record_statement( - context=context, - statement_id=statement_id, - sql=sql, - query_id=query_id, - status=status, - started_at=started_at, - completed_at=completed_at, - error=error, - ) diff --git a/src/sqlbuild/cost/constants.py b/src/sqlbuild/cost/constants.py index c426667edb..43664c4c4a 100644 --- a/src/sqlbuild/cost/constants.py +++ b/src/sqlbuild/cost/constants.py @@ -2,18 +2,5 @@ from decimal import Decimal -from sqlbuild.cost.classes.cost_telemetry_health import CostTelemetryHealth - DEFAULT_USD_PER_CREDIT: Decimal = Decimal("3.00") -DEFAULT_RATE_SOURCE: str = "default" -STANDARD_WAREHOUSE_TYPE: str = "STANDARD" -RUNNING_EXECUTION_STATUS: str = "RUNNING" SQLBUILD_QUERY_TAG_APP: str = "sqlbuild" -USD_PER_CREDIT_CONFIG_KEY: str = "usd_per_credit" -LATEST_COST_RUN_SELECTOR: str = "latest" -RUNNING_BUILD_STATUS: str = "running" -NO_WAREHOUSE_COMPUTE_REASON: str = "no_warehouse_compute" -MISSING_WAREHOUSE_METADATA_REASON: str = "missing_warehouse_metadata" -INCOMPLETE_HISTORY_REASON: str = "incomplete_history" -LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX: str = "Statement-ledger persistence failed (" -COST_TELEMETRY_HEALTH: CostTelemetryHealth = CostTelemetryHealth() diff --git a/src/sqlbuild/cost/exceptions.py b/src/sqlbuild/cost/exceptions.py deleted file mode 100644 index ba6a557195..0000000000 --- a/src/sqlbuild/cost/exceptions.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Cost observability errors.""" - - -class CostArtifactError(ValueError): - """Raised when a persisted cost artifact has an invalid shape.""" diff --git a/src/sqlbuild/cost/main/__init__.py b/src/sqlbuild/cost/main/__init__.py deleted file mode 100644 index 9b431179fa..0000000000 --- a/src/sqlbuild/cost/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Public cost observability entry points.""" diff --git a/src/sqlbuild/cost/main/collection.py b/src/sqlbuild/cost/main/collection.py deleted file mode 100644 index 392bba226d..0000000000 --- a/src/sqlbuild/cost/main/collection.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Snowflake cost collection entry point.""" - -from datetime import datetime -from decimal import Decimal -from pathlib import Path -from typing import Any - -from sqlbuild.cost._helpers.snowflake import collect_snowflake_run_cost -from sqlbuild.cost.models import RunCostSummary - - -def collect_snowflake_cost( - *, - connection: Any, - database: str | None = None, - run_id: str, - started_at: datetime, - completed_at: datetime, - statement_ledger_path: Path, - usd_per_credit: Decimal, - rate_source: str, - attempts: int = 3, - retry_delay_seconds: float = 1.0, -) -> RunCostSummary: - return collect_snowflake_run_cost( - connection=connection, - database=database, - run_id=run_id, - started_at=started_at, - completed_at=completed_at, - statement_ledger_path=statement_ledger_path, - usd_per_credit=usd_per_credit, - rate_source=rate_source, - attempts=attempts, - retry_delay_seconds=retry_delay_seconds, - ) diff --git a/src/sqlbuild/cursor_algebra/constants.py b/src/sqlbuild/cursor_algebra/constants.py index f3dac3ee9a..9d38c029db 100644 --- a/src/sqlbuild/cursor_algebra/constants.py +++ b/src/sqlbuild/cursor_algebra/constants.py @@ -11,14 +11,6 @@ CursorGrain.YEAR: 5, } -GRAIN_BATCH_SIZE: dict[CursorGrain, str] = { - CursorGrain.SECOND: "1s", - CursorGrain.MINUTE: "1m", - CursorGrain.HOUR: "1h", - CursorGrain.DAY: "1d", - CursorGrain.MONTH: "1mo", - CursorGrain.YEAR: "1y", -} DURATION_YEAR_UNIT: str = "y" DURATION_MONTH_UNIT: str = "mo" diff --git a/src/sqlbuild/diagnostics/_helpers/logging_formatters.py b/src/sqlbuild/diagnostics/_helpers/logging_formatters.py index 39a986cf75..5705251f17 100644 --- a/src/sqlbuild/diagnostics/_helpers/logging_formatters.py +++ b/src/sqlbuild/diagnostics/_helpers/logging_formatters.py @@ -4,7 +4,7 @@ import logging -from sqlbuild.presentation.classes.cli_style import CliStyle +from sqlbuild._native import CliStyle _LOGGER_ROOT_NAME: str = "sqlbuild" _INLINE_SQL_TRANSACTION_STATEMENTS: tuple[str, ...] = ("BEGIN", "COMMIT", "ROLLBACK") diff --git a/src/sqlbuild/diagnostics/classes/build_phase_timing_tracker.py b/src/sqlbuild/diagnostics/classes/build_phase_timing_tracker.py deleted file mode 100644 index dd9316c3aa..0000000000 --- a/src/sqlbuild/diagnostics/classes/build_phase_timing_tracker.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Invocation-scoped partial build timing tracker.""" - -from __future__ import annotations - -import time -from collections.abc import Callable, Iterator -from contextlib import contextmanager -from contextvars import ContextVar, Token -from typing import ClassVar - -from sqlbuild.diagnostics.models import PartialBuildPhaseTimings - - -class BuildPhaseTimingTracker: - """Retain disjoint monotonic phase durations across exceptional exits.""" - - def __init__(self, *, monotonic: Callable[[], float] = time.monotonic) -> None: - self._monotonic: Callable[[], float] = monotonic - self._started_at: float = monotonic() - self.compile_seconds: float | None = None - self.planning_seconds: float | None = None - self.connection_preparation_seconds: float | None = None - self.schema_preparation_seconds: float | None = None - self.execution_seconds: float | None = None - self.cost_collection_seconds: float | None = None - - def snapshot(self) -> PartialBuildPhaseTimings: - """Return currently available phase durations and total wall time.""" - - return PartialBuildPhaseTimings( - compile_seconds=self.compile_seconds, - planning_seconds=self.planning_seconds, - connection_preparation_seconds=self.connection_preparation_seconds, - schema_preparation_seconds=self.schema_preparation_seconds, - execution_seconds=self.execution_seconds, - cost_collection_seconds=self.cost_collection_seconds, - total_seconds=max(0.0, self._monotonic() - self._started_at), - ) - - @classmethod - def current(cls) -> BuildPhaseTimingTracker | None: - """Return the active build timing tracker.""" - - return cls._current.get() - - @contextmanager - def scope(self) -> Iterator[None]: - """Install this tracker for one command invocation and reset it afterward.""" - - token: Token[BuildPhaseTimingTracker | None] = self._current.set(self) - try: - yield - finally: - self._current.reset(token) - - _current: ClassVar[ContextVar[BuildPhaseTimingTracker | None]] = ContextVar( - "sqlbuild_build_phase_timing_tracker", default=None - ) diff --git a/src/sqlbuild/diagnostics/classes/diagnostics_console_formatter.py b/src/sqlbuild/diagnostics/classes/diagnostics_console_formatter.py index f09c973f14..2c1a556162 100644 --- a/src/sqlbuild/diagnostics/classes/diagnostics_console_formatter.py +++ b/src/sqlbuild/diagnostics/classes/diagnostics_console_formatter.py @@ -4,13 +4,13 @@ import logging +from sqlbuild._native import CliStyle from sqlbuild.diagnostics._helpers.logging_formatters import ( _append_exception, _format_console_header, _get_record_sql, _render_sql_inline, ) -from sqlbuild.presentation.classes.cli_style import CliStyle _SQL_SEPARATOR: str = "-" * 80 diff --git a/src/sqlbuild/diagnostics/classes/process_resource_tracker.py b/src/sqlbuild/diagnostics/classes/process_resource_tracker.py deleted file mode 100644 index 90fdb4a7ea..0000000000 --- a/src/sqlbuild/diagnostics/classes/process_resource_tracker.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Monotonic process resource tracker.""" - -from __future__ import annotations - -import time -from collections.abc import Callable - -from sqlbuild.diagnostics._helpers.process_resources import read_process_resources -from sqlbuild.diagnostics.models import ProcessResourceUsage - - -class ProcessResourceTracker: - """Capture wall and process resource deltas around one command.""" - - def __init__( - self, - *, - monotonic: Callable[[], float] = time.monotonic, - resource_reader: Callable[[], tuple[float, float, int | None]] = read_process_resources, - ) -> None: - self._monotonic: Callable[[], float] = monotonic - self._resource_reader: Callable[[], tuple[float, float, int | None]] = resource_reader - self._started_at: float = monotonic() - self._started_user, self._started_system, _ = resource_reader() - - def finish(self) -> ProcessResourceUsage: - """Return elapsed wall/CPU deltas and process peak RSS.""" - - user_cpu, system_cpu, max_rss_bytes = self._resource_reader() - return ProcessResourceUsage( - wall_seconds=max(0.0, self._monotonic() - self._started_at), - user_cpu_seconds=max(0.0, user_cpu - self._started_user), - system_cpu_seconds=max(0.0, system_cpu - self._started_system), - max_rss_bytes=max_rss_bytes, - ) diff --git a/src/sqlbuild/diagnostics/main/process_resource_reporting.py b/src/sqlbuild/diagnostics/main/process_resource_reporting.py index dd88f006e4..f5bcc28310 100644 --- a/src/sqlbuild/diagnostics/main/process_resource_reporting.py +++ b/src/sqlbuild/diagnostics/main/process_resource_reporting.py @@ -5,7 +5,7 @@ from collections.abc import Iterator from contextlib import contextmanager -from sqlbuild.diagnostics.classes.process_resource_tracker import ProcessResourceTracker +from sqlbuild._native import ProcessResourceTracker from sqlbuild.diagnostics.main._log_process_resources import log_process_resources from sqlbuild.diagnostics.models import ProcessResourceUsage diff --git a/src/sqlbuild/errors/contracts/main/__init__.py b/src/sqlbuild/errors/contracts/main/__init__.py deleted file mode 100644 index 3dafe77f29..0000000000 --- a/src/sqlbuild/errors/contracts/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Expected-error contract entries.""" diff --git a/src/sqlbuild/errors/contracts/main/error_code.py b/src/sqlbuild/errors/contracts/main/error_code.py deleted file mode 100644 index 5a686c4411..0000000000 --- a/src/sqlbuild/errors/contracts/main/error_code.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Expected-error code extraction entrypoint.""" - -from __future__ import annotations - - -def error_code(*, error: BaseException, fallback_code: str) -> str: - """Return a stable code from an expected exception-like object.""" - - return str(getattr(error, "code", fallback_code)) diff --git a/src/sqlbuild/errors/contracts/main/error_help.py b/src/sqlbuild/errors/contracts/main/error_help.py deleted file mode 100644 index f30417e028..0000000000 --- a/src/sqlbuild/errors/contracts/main/error_help.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Expected-error help extraction entrypoint.""" - -from __future__ import annotations - - -def error_help(error: BaseException) -> str | None: - """Return expected-error help text when present.""" - - help_text: object | None = getattr(error, "help", None) - return help_text if isinstance(help_text, str) else None diff --git a/src/sqlbuild/errors/contracts/main/error_message.py b/src/sqlbuild/errors/contracts/main/error_message.py deleted file mode 100644 index 0685ac3a46..0000000000 --- a/src/sqlbuild/errors/contracts/main/error_message.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Expected-error message extraction entrypoint.""" - -from __future__ import annotations - - -def error_message(error: BaseException) -> str: - """Return a stable message from an expected exception-like object.""" - - return str(getattr(error, "message", str(error))) diff --git a/src/sqlbuild/errors/setting_help/__init__.py b/src/sqlbuild/errors/setting_help/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/errors/setting_help/_helpers/__init__.py b/src/sqlbuild/errors/setting_help/_helpers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/errors/setting_help/_helpers/text.py b/src/sqlbuild/errors/setting_help/_helpers/text.py deleted file mode 100644 index 3bbe56e013..0000000000 --- a/src/sqlbuild/errors/setting_help/_helpers/text.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Exact TOML and MODEL-header snippets for diagnostics that name a setting.""" - -from __future__ import annotations - -from sqlbuild.errors.setting_help.constants import ( - ADDITIONAL_HELP_SEPARATOR, - HELP_CONTINUATION_INDENT, - SETTING_SNIPPET_INDENT, -) -from sqlbuild.errors.setting_help.types import SettingValue - - -def toml_value(value: SettingValue) -> str: - if isinstance(value, bool): - return "true" if value else "false" - if isinstance(value, int): - return str(value) - escaped: str = value.replace("\\", "\\\\").replace('"', '\\"') - return f'"{escaped}"' - - -def setting_snippet(*, section: str, key: str, value: SettingValue) -> str: - return "\n".join( - ( - f"{SETTING_SNIPPET_INDENT}[{section}]", - f"{SETTING_SNIPPET_INDENT}{key} = {toml_value(value)}", - ) - ) - - -def setting_help( - *, purpose: str, file_name: str, section: str, key: str, value: SettingValue -) -> str: - snippet: str = setting_snippet(section=section, key=key, value=value) - return f"{purpose}, set this in {file_name}:\n{snippet}" - - -def setting_note( - *, file_name: str, section: str, key: str, value: SettingValue, explicit: bool | None -) -> str: - rendered: str = toml_value(value) - if explicit is None: - return ( - f"the current value is [{section}] {key} = {rendered} (from {file_name} or its default)" - ) - if explicit: - return f"{file_name} sets [{section}] {key} = {rendered}" - return f"{file_name} does not set [{section}] {key}, so it defaults to {rendered}" - - -def model_header_help(*, purpose: str, entry: str, follow_up: str | None) -> str: - lines: list[str] = [ - f"{purpose}, add this to the MODEL header:", - f"{SETTING_SNIPPET_INDENT}MODEL (", - f"{SETTING_SNIPPET_INDENT} {entry},", - f"{SETTING_SNIPPET_INDENT} ...", - f"{SETTING_SNIPPET_INDENT});", - ] - if follow_up is not None: - lines.append(f"{HELP_CONTINUATION_INDENT}{follow_up}") - return "\n".join(lines) - - -def snippet_help(*, purpose: str, target: str, lines: tuple[str, ...]) -> str: - snippet: str = "\n".join(f"{SETTING_SNIPPET_INDENT}{line}" for line in lines) - return f"{purpose}, {target}:\n{snippet}" - - -def join_helps(helps: tuple[str, ...]) -> str: - return ADDITIONAL_HELP_SEPARATOR.join(helps) diff --git a/src/sqlbuild/errors/setting_help/constants.py b/src/sqlbuild/errors/setting_help/constants.py deleted file mode 100644 index 216f80e559..0000000000 --- a/src/sqlbuild/errors/setting_help/constants.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Layout of help text that shows an exact setting snippet.""" - -from __future__ import annotations - -SETTING_SNIPPET_INDENT: str = " " * 12 -HELP_CONTINUATION_INDENT: str = " " * 10 -ADDITIONAL_HELP_SEPARATOR: str = "\n = help: " diff --git a/src/sqlbuild/errors/setting_help/main/__init__.py b/src/sqlbuild/errors/setting_help/main/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/errors/setting_help/main/join_helps.py b/src/sqlbuild/errors/setting_help/main/join_helps.py deleted file mode 100644 index e63e7214a0..0000000000 --- a/src/sqlbuild/errors/setting_help/main/join_helps.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public entry for joining several help paragraphs.""" - -from __future__ import annotations - -from sqlbuild.errors.setting_help._helpers.text import join_helps as _join_helps - - -def join_helps(*helps: str) -> str: - """Join several help paragraphs into one help text, one `= help:` label per paragraph.""" - - return _join_helps(helps) diff --git a/src/sqlbuild/errors/setting_help/main/model_header_help.py b/src/sqlbuild/errors/setting_help/main/model_header_help.py deleted file mode 100644 index 8ad91217dd..0000000000 --- a/src/sqlbuild/errors/setting_help/main/model_header_help.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public entry for help that shows an exact MODEL header entry.""" - -from __future__ import annotations - -from sqlbuild.errors.setting_help._helpers.text import model_header_help as _model_header_help - - -def model_header_help(*, purpose: str, entry: str, follow_up: str | None = None) -> str: - """Render `, add this to the MODEL header:` followed by the exact header entry.""" - - return _model_header_help(purpose=purpose, entry=entry, follow_up=follow_up) diff --git a/src/sqlbuild/errors/setting_help/main/setting_help.py b/src/sqlbuild/errors/setting_help/main/setting_help.py deleted file mode 100644 index b802654af8..0000000000 --- a/src/sqlbuild/errors/setting_help/main/setting_help.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public entry for help that shows the exact setting to write.""" - -from __future__ import annotations - -from sqlbuild.errors.setting_help._helpers.text import setting_help as _setting_help -from sqlbuild.errors.setting_help.types import SettingValue - - -def setting_help( - *, purpose: str, file_name: str, section: str, key: str, value: SettingValue -) -> str: - """Render `, set this in :` followed by the exact TOML section and key.""" - - return _setting_help( - purpose=purpose, file_name=file_name, section=section, key=key, value=value - ) diff --git a/src/sqlbuild/errors/setting_help/main/setting_note.py b/src/sqlbuild/errors/setting_help/main/setting_note.py deleted file mode 100644 index 3bd64b34fe..0000000000 --- a/src/sqlbuild/errors/setting_help/main/setting_note.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public entry for the current-value note of a setting.""" - -from __future__ import annotations - -from sqlbuild.errors.setting_help._helpers.text import setting_note as _setting_note -from sqlbuild.errors.setting_help.types import SettingValue - - -def setting_note( - *, file_name: str, section: str, key: str, value: SettingValue, explicit: bool | None = True -) -> str: - """Render a setting's current value; `explicit=None` when its provenance is unknown.""" - - return _setting_note( - file_name=file_name, section=section, key=key, value=value, explicit=explicit - ) diff --git a/src/sqlbuild/errors/setting_help/main/snippet_help.py b/src/sqlbuild/errors/setting_help/main/snippet_help.py deleted file mode 100644 index 6f5025d24a..0000000000 --- a/src/sqlbuild/errors/setting_help/main/snippet_help.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public entry for help that shows an exact authored snippet to add.""" - -from __future__ import annotations - -from sqlbuild.errors.setting_help._helpers.text import snippet_help as _snippet_help - - -def snippet_help(*, purpose: str, target: str, lines: tuple[str, ...]) -> str: - """Render `, :` followed by the exact lines to write, one per line.""" - - return _snippet_help(purpose=purpose, target=target, lines=lines) diff --git a/src/sqlbuild/errors/setting_help/types.py b/src/sqlbuild/errors/setting_help/types.py deleted file mode 100644 index f3060d3fb9..0000000000 --- a/src/sqlbuild/errors/setting_help/types.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Types for setting help text.""" - -from __future__ import annotations - -type SettingValue = bool | int | str diff --git a/src/sqlbuild/executor/auditing/_helpers/execution.py b/src/sqlbuild/executor/auditing/_helpers/execution.py index e24ee76f7f..6b16e9c4a3 100644 --- a/src/sqlbuild/executor/auditing/_helpers/execution.py +++ b/src/sqlbuild/executor/auditing/_helpers/execution.py @@ -10,6 +10,7 @@ from typing import Any, cast from sqlbuild._native import ( + CostContext, assert_no_unresolved_sql_markers, evaluate_measurement, render_audit_sql, @@ -22,7 +23,6 @@ ) from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.planner.models import AuditPlanEntry -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.constants import ( DEFAULT_EVIDENCE_ROW_LIMIT, diff --git a/src/sqlbuild/executor/build/_helpers/source_node.py b/src/sqlbuild/executor/build/_helpers/source_node.py index ce74485e38..028b2af56b 100644 --- a/src/sqlbuild/executor/build/_helpers/source_node.py +++ b/src/sqlbuild/executor/build/_helpers/source_node.py @@ -7,12 +7,12 @@ from collections.abc import Callable from typing import Any +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.planner.models import PlanOutput, SourceLoadPlanEntry -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.executor.build.models import BuildCallbacks, BuildRuntimeParams from sqlbuild.executor.load.main._execute import execute_source_load from sqlbuild.executor.load.models import LoadExecutionResult, LoadRuntimeParams diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 220e1bf020..892158d337 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -15,6 +15,7 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import RelationInfo @@ -37,7 +38,6 @@ IncrementalStrategy, PlanAction, ) -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.diagnostics.main.log_debug_event import log_debug_event diff --git a/src/sqlbuild/executor/clone/_helpers/prephase_progress.py b/src/sqlbuild/executor/clone/_helpers/prephase_progress.py index 5276e7a13f..9ff8ce45cc 100644 --- a/src/sqlbuild/executor/clone/_helpers/prephase_progress.py +++ b/src/sqlbuild/executor/clone/_helpers/prephase_progress.py @@ -9,13 +9,13 @@ from functools import partial from typing import TextIO +from sqlbuild._native import CliStyle from sqlbuild.executor.clone.classes.prephase_progress_reporter import ( ClonePrephaseProgressReporter, ) from sqlbuild.executor.clone.classes.progress_state import CloneProgressState from sqlbuild.executor.clone.models import CloneItemResult, PrephaseProgressRow from sqlbuild.executor.clone.types import CloneAction, CloneItemCallback, CloneStatus -from sqlbuild.presentation.classes.cli_style import CliStyle _TYPE_WIDTH: int = 10 _NAME_WIDTH: int = 40 diff --git a/src/sqlbuild/executor/functions/main/_execute.py b/src/sqlbuild/executor/functions/main/_execute.py index b90c1b264e..f00bba06d6 100644 --- a/src/sqlbuild/executor/functions/main/_execute.py +++ b/src/sqlbuild/executor/functions/main/_execute.py @@ -8,6 +8,9 @@ from sqlbuild._native import ( compute_function_definition_hash, + error_code, + error_help, + error_message, function_node_type, write_fingerprint, ) @@ -18,9 +21,6 @@ from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import FunctionPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.errors.contracts.main.error_code import error_code -from sqlbuild.errors.contracts.main.error_help import error_help -from sqlbuild.errors.contracts.main.error_message import error_message from sqlbuild.executor.build.models import FunctionExecutionResult from sqlbuild.executor.functions.constants import ( FUNCTION_EXECUTION_FAILED_CODE, diff --git a/src/sqlbuild/executor/janitor/_helpers/classification.py b/src/sqlbuild/executor/janitor/_helpers/classification.py index 2fbb4127c4..63bc113985 100644 --- a/src/sqlbuild/executor/janitor/_helpers/classification.py +++ b/src/sqlbuild/executor/janitor/_helpers/classification.py @@ -6,14 +6,12 @@ from fnmatch import fnmatchcase from typing import Any +from sqlbuild._native import is_scenario_artifact_physical_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.planner.main.scenarios.is_scenario_artifact_physical_name import ( - is_scenario_artifact_physical_name, -) from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.executor.diff.classes.query_artifact_lifecycle import QueryDiffArtifactLifecycle from sqlbuild.executor.diff.models import QueryDiffArtifactInspection diff --git a/src/sqlbuild/executor/load/main/_execute.py b/src/sqlbuild/executor/load/main/_execute.py index ea10e1add6..4db2a54708 100644 --- a/src/sqlbuild/executor/load/main/_execute.py +++ b/src/sqlbuild/executor/load/main/_execute.py @@ -7,13 +7,13 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load._helpers.cursors import ( diff --git a/src/sqlbuild/executor/pipeline/_helpers/auditing.py b/src/sqlbuild/executor/pipeline/_helpers/auditing.py index f645564581..aee09c8380 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/auditing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/auditing.py @@ -15,10 +15,10 @@ from typing import Any, cast from uuid import uuid4 +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.main._execute import execute_audit diff --git a/src/sqlbuild/executor/pipeline/_helpers/connections.py b/src/sqlbuild/executor/pipeline/_helpers/connections.py index 90cc7ae2b2..adcbb3d091 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/connections.py +++ b/src/sqlbuild/executor/pipeline/_helpers/connections.py @@ -8,9 +8,9 @@ from contextvars import copy_context from typing import Any +from sqlbuild._native import BuildPhaseTimingTracker from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker from sqlbuild.executor.build.models import BuildCallbacks from sqlbuild.executor.pipeline._helpers.schema_preparation import prepare_build_schemas from sqlbuild.executor.pipeline.models import BuildConnectionPreparation diff --git a/src/sqlbuild/executor/pipeline/_helpers/scenario.py b/src/sqlbuild/executor/pipeline/_helpers/scenario.py index a7efed0c55..60f819b29f 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/scenario.py +++ b/src/sqlbuild/executor/pipeline/_helpers/scenario.py @@ -18,19 +18,19 @@ from sqlbuild._native import ( NativeProgressProjector, current_native_progress_projector, + error_code, + error_help, + error_message, scenario_local_type_overrides_for_dialect, ) +from sqlbuild._native import build_cli_scenario_plan as build_scenario_plan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.main.scenarios.scenario import build_scenario_plan from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.diagnostics.main.log_debug_event import log_debug_event -from sqlbuild.errors.contracts.main.error_code import error_code -from sqlbuild.errors.contracts.main.error_help import error_help -from sqlbuild.errors.contracts.main.error_message import error_message from sqlbuild.executor.pipeline._helpers.connections import ( close_connections, open_worker_connections, diff --git a/src/sqlbuild/executor/pipeline/main/run.py b/src/sqlbuild/executor/pipeline/main/run.py index 8ed47a4f25..e2b61b7ec7 100644 --- a/src/sqlbuild/executor/pipeline/main/run.py +++ b/src/sqlbuild/executor/pipeline/main/run.py @@ -6,10 +6,9 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import BuildPhaseTimingTracker, CostContext from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.cost.classes.cost_context import CostContext -from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.main._external_source_loads import ( diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index 0a291eb538..1c36b1e9dd 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -8,6 +8,7 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.discovery.models import DiscoveredAssetFunction from sqlbuild.compiler.python_nodes.types import ( @@ -15,7 +16,6 @@ PythonNodeKind, PythonNodeStatus, ) -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store from sqlbuild.executor.node_results.models import NodeResultRecord diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index 64d67e6450..3faa8e9f0e 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -8,6 +8,7 @@ from functools import partial from typing import Any +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.discovery.models import ( DiscoveredAssetFunction, @@ -20,7 +21,6 @@ PythonSqlRunLifecyclePlan, ) from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.main._execute import execute_source_load from sqlbuild.executor.load.main._resource_kind import load_resource_kind diff --git a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py index 86f89689f4..fe5a64ac54 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py @@ -6,6 +6,7 @@ from collections.abc import Mapping from typing import Any +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction @@ -15,7 +16,6 @@ PythonNodeIdentity, ) from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store diff --git a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py index 82c964c868..bed168a6ee 100644 --- a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py +++ b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py @@ -5,10 +5,10 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.python_nodes.models import DiscoveredPythonNode, PythonNodeGraph from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store from sqlbuild.executor.python_nodes._helpers.fingerprinting import ( try_write_python_node_identity_fingerprint, diff --git a/src/sqlbuild/executor/run/_helpers/execution/hooks.py b/src/sqlbuild/executor/run/_helpers/execution/hooks.py index 0d60c85c5d..e5cd6aa0b1 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/hooks.py +++ b/src/sqlbuild/executor/run/_helpers/execution/hooks.py @@ -5,7 +5,7 @@ from collections.abc import Mapping from typing import Any -from sqlbuild._native import render_source_relation +from sqlbuild._native import CostContext, render_source_relation from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( @@ -22,7 +22,6 @@ from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard diff --git a/src/sqlbuild/executor/run/_helpers/execution/results.py b/src/sqlbuild/executor/run/_helpers/execution/results.py index 3266c9e301..502bb15301 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/results.py +++ b/src/sqlbuild/executor/run/_helpers/execution/results.py @@ -4,7 +4,7 @@ from dataclasses import replace -from sqlbuild._native import format_waiting_on_empty_inputs +from sqlbuild._native import error_code, error_help, error_message, format_waiting_on_empty_inputs from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.planner.models import ( CursorBounds, @@ -12,9 +12,6 @@ MaximumStartSafetyEvidence, ModelPlanEntry, ) -from sqlbuild.errors.contracts.main.error_code import error_code -from sqlbuild.errors.contracts.main.error_help import error_help -from sqlbuild.errors.contracts.main.error_message import error_message from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run.constants import ( RUN_AUDIT_FAILED_CODE, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index c7061b769f..9b0d288808 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -6,6 +6,7 @@ from typing import Any from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token +from sqlbuild._native import future_cursor_cap_warning, maximum_start_cap_warning from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo @@ -17,8 +18,6 @@ resolve_relation_location_qualified_name, ) from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.planner.main.execution.future_cursor_warning import future_cursor_cap_warning -from sqlbuild.compiler.planner.main.execution.maximum_start_warning import maximum_start_cap_warning from sqlbuild.compiler.planner.models import CursorBounds, ModelPlanEntry from sqlbuild.compiler.planner.types import IncrementalStrategy, OnSchemaChange from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index 8d0eba4ad8..2fdb534834 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -15,7 +15,10 @@ aligned_timestamp_bounds, compute_query_hash, cursor_bound_display, + empty_cursor_window, + future_cursor_cap_warning, inclusive_cursor_end, + maximum_start_cap_warning, microbatch_limit_warning, resolve_effective_microbatch_batch_size, ) @@ -42,9 +45,6 @@ from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) -from sqlbuild.compiler.planner.main.cursor_window.empty_cursor_window import empty_cursor_window -from sqlbuild.compiler.planner.main.execution.future_cursor_warning import future_cursor_cap_warning -from sqlbuild.compiler.planner.main.execution.maximum_start_warning import maximum_start_cap_warning from sqlbuild.compiler.planner.models import ( CursorBounds, Duration, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py b/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py index 79b37bd434..d8eb5b41a5 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py @@ -4,12 +4,12 @@ from dataclasses import replace -from sqlbuild._native import replaces_incremental_relation -from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists -from sqlbuild.compiler.planner.main.cursor_window.empty_cursor_window import empty_cursor_window -from sqlbuild.compiler.planner.main.execution.future_cursor_warning import ( +from sqlbuild._native import ( + empty_cursor_window, future_cursor_cap_warning, + replaces_incremental_relation, ) +from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists from sqlbuild.compiler.planner.models import CursorBounds, ModelPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( diff --git a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py index e611499f86..edd3b81662 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py +++ b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py @@ -8,6 +8,14 @@ from functools import partial from typing import Any +from sqlbuild._native import ( + apply_maximum_start_policy, + apply_typed_cursor_replay_policy, + cursor_window_override_flags, + decide_empty_cursor_inputs, + empty_input_rebuild_refusal, + resolve_cursor_intrinsics, +) from sqlbuild._native import cursor_clamp as clamp from sqlbuild._native import cursor_compare as compare from sqlbuild._native import ( @@ -22,24 +30,10 @@ from sqlbuild._native import cursor_render as render from sqlbuild._native import cursor_sentinel_from_token as sentinel_from_token from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild._native import resolve_cursor_intrinsics +from sqlbuild._native import resolve_future_cursor_safety as apply_future_cursor_safety from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL -from sqlbuild.compiler.planner.main.cursor_window.cursor_window_override_flags import ( - cursor_window_override_flags, -) -from sqlbuild.compiler.planner.main.cursor_window.empty_cursor_input_decision import ( - decide_empty_cursor_inputs, -) -from sqlbuild.compiler.planner.main.cursor_window.empty_input_rebuild_refusal import ( - empty_input_rebuild_refusal, -) -from sqlbuild.compiler.planner.main.execution.cursor_replay_policy import ( - apply_typed_cursor_replay_policy, -) -from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety -from sqlbuild.compiler.planner.main.execution.maximum_start_safety import apply_maximum_start_policy from sqlbuild.compiler.planner.models import ( CursorBounds, CursorInputEvidence, diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index 9843ae0a42..f4a286b14f 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -5,6 +5,7 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import setting_help from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo @@ -19,7 +20,6 @@ from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.errors.setting_help.main.setting_help import setting_help from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.execution.final_audits import run_final_model_audits from sqlbuild.executor.run._helpers.execution.hook_phases import ( diff --git a/src/sqlbuild/executor/scenario/_helpers/local/execution.py b/src/sqlbuild/executor/scenario/_helpers/local/execution.py index d3173a0a38..90a20fb144 100644 --- a/src/sqlbuild/executor/scenario/_helpers/local/execution.py +++ b/src/sqlbuild/executor/scenario/_helpers/local/execution.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any +from sqlbuild._native import error_code, error_help, error_message from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.types import TablePromotionMode @@ -20,9 +21,6 @@ ScenarioRelationPlan, ) from sqlbuild.compiler.planner.types import ScenarioArtifactKind -from sqlbuild.errors.contracts.main.error_code import error_code -from sqlbuild.errors.contracts.main.error_help import error_help -from sqlbuild.errors.contracts.main.error_message import error_message from sqlbuild.executor.build.models import FunctionExecutionResult from sqlbuild.executor.functions.main._execute import execute_function from sqlbuild.executor.run.models import ModelExecutionResult diff --git a/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py b/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py index b2a1233172..e24624c68a 100644 --- a/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py +++ b/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py @@ -5,10 +5,9 @@ from pathlib import Path from typing import Any +from sqlbuild._native import error_code, error_message from sqlbuild.compiler.planner.types import ScenarioArtifactKind from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.errors.contracts.main.error_code import error_code -from sqlbuild.errors.contracts.main.error_message import error_message from sqlbuild.executor.scenario._helpers.snapshots.core import ( is_scenario_snapshot_fresh, read_scenario_snapshot_jsonl, diff --git a/src/sqlbuild/executor/scenario/_helpers/snapshots/core.py b/src/sqlbuild/executor/scenario/_helpers/snapshots/core.py index 3d276078dd..78e7096b66 100644 --- a/src/sqlbuild/executor/scenario/_helpers/snapshots/core.py +++ b/src/sqlbuild/executor/scenario/_helpers/snapshots/core.py @@ -11,6 +11,7 @@ from typing import Any from uuid import UUID +from sqlbuild._native import error_code, error_message from sqlbuild.compiler.planner.models import ( ScenarioExecutionPlan, ScenarioFixturePlan, @@ -18,8 +19,6 @@ ) from sqlbuild.compiler.planner.types import ScenarioArtifactKind from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.errors.contracts.main.error_code import error_code -from sqlbuild.errors.contracts.main.error_message import error_message from sqlbuild.executor.scenario.constants import ( JSONL_BLANK_LINE, SCENARIO_EXEC_CAPTURE_LIMIT_EXCEEDED, diff --git a/src/sqlbuild/executor/scenario/classes/scenario_interrupts.py b/src/sqlbuild/executor/scenario/classes/scenario_interrupts.py index c6a1829b3c..33874f98b6 100644 --- a/src/sqlbuild/executor/scenario/classes/scenario_interrupts.py +++ b/src/sqlbuild/executor/scenario/classes/scenario_interrupts.py @@ -8,9 +8,7 @@ from collections.abc import Callable, Iterator from contextlib import contextmanager -from sqlbuild.presentation.main.transient_line_coordinator import ( - shared_transient_line_coordinator, -) +from sqlbuild._native import shared_transient_line_coordinator _STOP_NOTICE: str = "Interrupted; cleaning up running scenarios...\n" _ABANDON_NOTICE: str = ( diff --git a/src/sqlbuild/executor/scenario/main/_capture.py b/src/sqlbuild/executor/scenario/main/_capture.py index 7cd3dbbbaa..568a1e995c 100644 --- a/src/sqlbuild/executor/scenario/main/_capture.py +++ b/src/sqlbuild/executor/scenario/main/_capture.py @@ -4,8 +4,8 @@ from typing import Any +from sqlbuild._native import error_code from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.errors.contracts.main.error_code import error_code from sqlbuild.executor.scenario._helpers.capture.relation import ( capture_scenario_snapshot_relation, scenario_capture_source_relation_name, diff --git a/src/sqlbuild/integrations/dbt/constants.py b/src/sqlbuild/integrations/dbt/constants.py deleted file mode 100644 index e6b2f3d9d6..0000000000 --- a/src/sqlbuild/integrations/dbt/constants.py +++ /dev/null @@ -1,20 +0,0 @@ -"""dbt integration constants.""" - -from sqlbuild.integrations.dbt.types import DbtInteropCommand - -DBT_EXECUTION_COMMANDS: frozenset[DbtInteropCommand] = frozenset( - ( - DbtInteropCommand.PLAN, - DbtInteropCommand.RUN, - DbtInteropCommand.BUILD, - ) -) -DBT_EXECUTION_DISPLAY_FLAGS: frozenset[str] = frozenset(("--json", "--verbose", "-v")) - -DBT_MANIFEST_CONFIG_KEY: str = "config" -DBT_MANIFEST_MATERIALIZED_KEY: str = "materialized" - - -DBT_DEFER_FLAG: str = "--defer" -DBT_FULL_REFRESH_FLAG: str = "--full-refresh" -DBT_UNIQUE_ID_SEPARATOR: str = "." diff --git a/src/sqlbuild/integrations/dbt/models.py b/src/sqlbuild/integrations/dbt/models.py index 6635c9274a..742e719653 100644 --- a/src/sqlbuild/integrations/dbt/models.py +++ b/src/sqlbuild/integrations/dbt/models.py @@ -165,18 +165,6 @@ class NormalizedDbtProfileConnection: warnings: tuple[str, ...] = field(default_factory=tuple) -@dataclass(frozen=True) -class DbtProfileConnectionRequest: - """Request to resolve one dbt-profile-backed SQLBuild connection.""" - - sqlbuild_project_dir: Path - dbt_project_dir: Path | None - profiles_dir: Path | None - profile_name: str | None - target_name: str | None - cli_vars: dict[str, object] = field(default_factory=dict) - - @dataclass(frozen=True) class DbtInitProgressCallbacks: """Optional progress hooks for `sqb dbt init` phases.""" diff --git a/src/sqlbuild/integrations/dbt/types.py b/src/sqlbuild/integrations/dbt/types.py index c9558180ba..d1de84b07e 100644 --- a/src/sqlbuild/integrations/dbt/types.py +++ b/src/sqlbuild/integrations/dbt/types.py @@ -13,17 +13,6 @@ class DbtInvoker(Protocol): def __call__(self, *, argv: tuple[str, ...], cwd: Path | None) -> object: ... -class DbtSupportedResourceType(StrEnum): - """dbt resource types SQLBuild currently handles with dedicated behavior.""" - - MODEL = "model" - SEED = "seed" - SNAPSHOT = "snapshot" - SOURCE = "source" - TEST = "test" - UNIT_TEST = "unit_test" - - class DbtInteropCommand(StrEnum): """dbt interop commands with SQLBuild participation.""" diff --git a/src/sqlbuild/lint/_helpers/__init__.py b/src/sqlbuild/lint/_helpers/__init__.py deleted file mode 100644 index 70254bb5af..0000000000 --- a/src/sqlbuild/lint/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Helpers for the lint and format layer.""" diff --git a/src/sqlbuild/lint/classes/__init__.py b/src/sqlbuild/lint/classes/__init__.py deleted file mode 100644 index 29801d9b58..0000000000 --- a/src/sqlbuild/lint/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Runtime classes for the lint and format layer.""" diff --git a/src/sqlbuild/lint/classes/stop_context.py b/src/sqlbuild/lint/classes/stop_context.py deleted file mode 100644 index 3b6215ebd6..0000000000 --- a/src/sqlbuild/lint/classes/stop_context.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Thread-local stop signal for speculative lint runs.""" - -from contextvars import ContextVar -from threading import Event -from typing import ClassVar - -from sqlbuild.lint.exceptions import LintStoppedError - - -class LintStopContext: - """Own the stop signal that lets a speculative lint run be abandoned between files.""" - - active: ClassVar[ContextVar[Event | None]] = ContextVar("active_lint_stop", default=None) - - @staticmethod - def check() -> None: - """Abandon a speculative lint run on this thread once its owner no longer needs it.""" - - stop: Event | None = LintStopContext.active.get() - if stop is not None and stop.is_set(): - raise LintStoppedError diff --git a/src/sqlbuild/lint/main/__init__.py b/src/sqlbuild/lint/main/__init__.py deleted file mode 100644 index 43780eb795..0000000000 --- a/src/sqlbuild/lint/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Orchestration for the lint and format layer.""" diff --git a/src/sqlbuild/presentation/_helpers/__init__.py b/src/sqlbuild/presentation/_helpers/__init__.py deleted file mode 100644 index c43f9eed92..0000000000 --- a/src/sqlbuild/presentation/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Presentation implementations.""" diff --git a/src/sqlbuild/presentation/_helpers/structure.py b/src/sqlbuild/presentation/_helpers/structure.py deleted file mode 100644 index dfe6e82998..0000000000 --- a/src/sqlbuild/presentation/_helpers/structure.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Structural CLI vocabulary implementations (trees, phase lines, headers).""" - -from __future__ import annotations - -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.constants import ( - TREE_BRANCH_GLYPH, - TREE_LAST_GLYPH, -) - - -def tree_connector(*, style: CliStyle, last: bool) -> str: - """Render a dim tree connector for a group entry.""" - - return style.muted(TREE_LAST_GLYPH if last else TREE_BRANCH_GLYPH) - - -def format_surface_header(*, style: CliStyle, title: str, context: str | None = None) -> str: - """Render a command surface header: accent title plus dim context.""" - - rendered: str = style.title(title) - if context: - rendered = f"{rendered} {style.muted(context)}" - return rendered diff --git a/src/sqlbuild/presentation/_helpers/terminal_capabilities.py b/src/sqlbuild/presentation/_helpers/terminal_capabilities.py deleted file mode 100644 index ec3b5e3cfa..0000000000 --- a/src/sqlbuild/presentation/_helpers/terminal_capabilities.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Terminal capability implementations.""" - -from __future__ import annotations - -import shutil - -_DEFAULT_TERMINAL_COLUMNS: int = 120 - - -def terminal_columns() -> int: - """Return the current terminal width in columns.""" - - return shutil.get_terminal_size(fallback=(_DEFAULT_TERMINAL_COLUMNS, 24)).columns diff --git a/src/sqlbuild/presentation/_helpers/transient_line.py b/src/sqlbuild/presentation/_helpers/transient_line.py deleted file mode 100644 index 7a74e98f60..0000000000 --- a/src/sqlbuild/presentation/_helpers/transient_line.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Process-wide transient terminal line coordination.""" - -from __future__ import annotations - -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator - -_SHARED_COORDINATOR: TransientLineCoordinator = TransientLineCoordinator() - - -def shared_transient_line_coordinator() -> TransientLineCoordinator: - """Return the coordinator shared by every terminal writer in this process.""" - - return _SHARED_COORDINATOR diff --git a/src/sqlbuild/presentation/classes/__init__.py b/src/sqlbuild/presentation/classes/__init__.py deleted file mode 100644 index da0b1c0690..0000000000 --- a/src/sqlbuild/presentation/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Presentation classes.""" diff --git a/src/sqlbuild/presentation/classes/cli_style.py b/src/sqlbuild/presentation/classes/cli_style.py deleted file mode 100644 index 4981b60221..0000000000 --- a/src/sqlbuild/presentation/classes/cli_style.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Semantic styling class for human CLI output.""" - -from __future__ import annotations - -from sqlbuild.presentation.constants import ( - ERROR_STATUS_LABELS, - SKIPPED_STATUS_LABELS, - SUCCESS_STATUS_LABELS, - WARNING_STATUS_LABELS, -) -from sqlbuild.presentation.models import CliTheme, TextStyle - -_BOLD: str = "\033[1m" -_DIM: str = "\033[2m" -_RED: str = "\033[38;5;167m" -_GREEN: str = "\033[32m" -_YELLOW: str = "\033[33m" -_BLUE: str = "\033[34m" -_ORANGE: str = "\033[38;5;208m" - -default_cli_theme: CliTheme = CliTheme( - title=TextStyle(_BLUE + _BOLD), - section=TextStyle(_BOLD), - label=TextStyle(_DIM), - value=TextStyle(_BLUE), - accent=TextStyle(_BLUE), - accent_strong=TextStyle(_BLUE + _BOLD), - plan_section=TextStyle(_BOLD), - object_name=TextStyle(""), - command=TextStyle(_DIM), - success=TextStyle(_GREEN), - success_strong=TextStyle(_GREEN + _BOLD), - warning=TextStyle(_YELLOW), - warning_strong=TextStyle(_YELLOW + _BOLD), - error=TextStyle(_RED), - error_strong=TextStyle(_RED + _BOLD), - error_muted=TextStyle(_RED + _DIM), - log_label=TextStyle(_DIM), - skipped=TextStyle(_DIM), - muted=TextStyle(_DIM), - dbt_section=TextStyle(_BOLD), - dbt_label=TextStyle(_ORANGE), - dbt_object_name=TextStyle(""), - dbt_execution_label=TextStyle(_ORANGE), -) - - -class CliStyle: - """Semantic CLI styling facade used by human-output formatters.""" - - def __init__(self, *, use_color: bool, theme: CliTheme = default_cli_theme) -> None: - self.use_color: bool = use_color - self.theme: CliTheme = theme - - def title(self, text: str) -> str: - return self.theme.title.apply(text=text, use_color=self.use_color) - - def section(self, text: str) -> str: - return self.theme.section.apply(text=text, use_color=self.use_color) - - def label(self, text: str) -> str: - return self.theme.label.apply(text=text, use_color=self.use_color) - - def value(self, text: str) -> str: - return self.theme.value.apply(text=text, use_color=self.use_color) - - def accent(self, text: str) -> str: - return self.theme.accent.apply(text=text, use_color=self.use_color) - - def accent_strong(self, text: str) -> str: - return self.theme.accent_strong.apply(text=text, use_color=self.use_color) - - def plan_section(self, text: str) -> str: - return self.theme.plan_section.apply(text=text, use_color=self.use_color) - - def object_name(self, text: str) -> str: - return self.theme.object_name.apply(text=text, use_color=self.use_color) - - def command(self, text: str) -> str: - return self.theme.command.apply(text=text, use_color=self.use_color) - - def muted(self, text: str) -> str: - return self.theme.muted.apply(text=text, use_color=self.use_color) - - def success(self, text: str) -> str: - return self.theme.success.apply(text=text, use_color=self.use_color) - - def success_strong(self, text: str) -> str: - return self.theme.success_strong.apply(text=text, use_color=self.use_color) - - def warning(self, text: str) -> str: - return self.theme.warning.apply(text=text, use_color=self.use_color) - - def warning_strong(self, text: str) -> str: - return self.theme.warning_strong.apply(text=text, use_color=self.use_color) - - def error(self, text: str) -> str: - return self.theme.error.apply(text=text, use_color=self.use_color) - - def error_strong(self, text: str) -> str: - return self.theme.error_strong.apply(text=text, use_color=self.use_color) - - def error_muted(self, text: str) -> str: - return self.theme.error_muted.apply(text=text, use_color=self.use_color) - - def log_label(self, text: str) -> str: - return self.theme.log_label.apply(text=text, use_color=self.use_color) - - def dbt_section(self, text: str) -> str: - return self.theme.dbt_section.apply(text=text, use_color=self.use_color) - - def dbt_label(self, text: str) -> str: - return self.theme.dbt_label.apply(text=text, use_color=self.use_color) - - def dbt_object_name(self, text: str) -> str: - return self.theme.dbt_object_name.apply(text=text, use_color=self.use_color) - - def dbt_execution_label(self, text: str) -> str: - return self.theme.dbt_execution_label.apply(text=text, use_color=self.use_color) - - def status(self, *, status: str, text: str | None = None) -> str: - """Style status text according to the status word.""" - - rendered: str = status if text is None else text - normalized: str = status.upper() - if normalized in SUCCESS_STATUS_LABELS: - return self.theme.success.apply(text=rendered, use_color=self.use_color) - if normalized in WARNING_STATUS_LABELS: - return self.theme.warning.apply(text=rendered, use_color=self.use_color) - if normalized in ERROR_STATUS_LABELS: - return self.theme.error.apply(text=rendered, use_color=self.use_color) - if normalized in SKIPPED_STATUS_LABELS: - return self.theme.skipped.apply(text=rendered, use_color=self.use_color) - return rendered diff --git a/src/sqlbuild/presentation/classes/transient_line_coordinator.py b/src/sqlbuild/presentation/classes/transient_line_coordinator.py deleted file mode 100644 index 9975f4cf9a..0000000000 --- a/src/sqlbuild/presentation/classes/transient_line_coordinator.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Coordinator for one live transient terminal line and persistent writes.""" - -from __future__ import annotations - -import threading -from typing import TextIO - -from sqlbuild.presentation.types import TransientLineOwner - - -class TransientLineCoordinator: - """Keep persistent terminal lines from landing on a live spinner or status line.""" - - def __init__(self) -> None: - self.lock: threading.RLock = threading.RLock() - self._active: tuple[TextIO, TransientLineOwner] | None = None - - def claim(self, *, stream: TextIO, owner: TransientLineOwner) -> None: - """Record the owner of the transient line currently drawn on a terminal stream.""" - - with self.lock: - self._active = (stream, owner) - - def release(self, *, owner: TransientLineOwner) -> None: - """Forget a transient owner once its line is no longer drawn.""" - - with self.lock: - if self._active is not None and self._active[1] is owner: - self._active = None - - def clear_active(self) -> None: - """Erase and forget any live transient line, so nothing redraws it afterwards.""" - - with self.lock: - if self._active is None: - return - owner: TransientLineOwner = self._active[1] - self._active = None - owner.clear_transient_line() - - def write_persistent(self, *, stream: TextIO, text: str) -> None: - """Write lasting text, clearing and then redrawing any live transient line.""" - - with self.lock: - owner: TransientLineOwner | None = self._interrupted_owner(stream=stream) - if owner is not None: - owner.clear_transient_line() - stream.write(text) - stream.flush() - if owner is not None: - owner.redraw_transient_line() - - def _interrupted_owner(self, *, stream: TextIO) -> TransientLineOwner | None: - if self._active is None: - return None - owner_stream: TextIO - owner: TransientLineOwner - owner_stream, owner = self._active - if owner_stream is stream or (_is_tty(owner_stream) and _is_tty(stream)): - return owner - return None - - -def _is_tty(stream: TextIO) -> bool: - return hasattr(stream, "isatty") and stream.isatty() diff --git a/src/sqlbuild/presentation/constants.py b/src/sqlbuild/presentation/constants.py index a06117ca75..ef8eccfe8f 100644 --- a/src/sqlbuild/presentation/constants.py +++ b/src/sqlbuild/presentation/constants.py @@ -3,11 +3,3 @@ from __future__ import annotations DEFAULT_MAX_DISPLAY_ENTRIES: int = 20 -ERROR_STATUS_LABELS: frozenset[str] = frozenset({"ERROR", "FAIL", "FAILED"}) -PHASE_FAIL_GLYPH: str = "\u2717" -PHASE_OK_GLYPH: str = "\u2713" -SKIPPED_STATUS_LABELS: frozenset[str] = frozenset({"SKIP", "SKIPPED"}) -SUCCESS_STATUS_LABELS: frozenset[str] = frozenset({"OK", "PASS", "SUCCESS"}) -TREE_BRANCH_GLYPH: str = "\u251c\u2500\u2500" -TREE_LAST_GLYPH: str = "\u2514\u2500\u2500" -WARNING_STATUS_LABELS: frozenset[str] = frozenset({"WARN", "WARNING"}) diff --git a/src/sqlbuild/presentation/main/__init__.py b/src/sqlbuild/presentation/main/__init__.py deleted file mode 100644 index 48e5caacda..0000000000 --- a/src/sqlbuild/presentation/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Presentation function entries.""" diff --git a/src/sqlbuild/presentation/main/terminal_columns.py b/src/sqlbuild/presentation/main/terminal_columns.py deleted file mode 100644 index cbbfdd61a2..0000000000 --- a/src/sqlbuild/presentation/main/terminal_columns.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public terminal width entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.terminal_capabilities import ( - terminal_columns as _terminal_columns, -) - - -def terminal_columns() -> int: - """Return the current terminal width in columns.""" - - return _terminal_columns() diff --git a/src/sqlbuild/presentation/main/transient_line_coordinator.py b/src/sqlbuild/presentation/main/transient_line_coordinator.py deleted file mode 100644 index 8a97433a08..0000000000 --- a/src/sqlbuild/presentation/main/transient_line_coordinator.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public shared transient line coordinator entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.transient_line import ( - shared_transient_line_coordinator as _shared_transient_line_coordinator, -) -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator - - -def shared_transient_line_coordinator() -> TransientLineCoordinator: - """Return the coordinator shared by every terminal writer in this process.""" - - return _shared_transient_line_coordinator() diff --git a/src/sqlbuild/presentation/models.py b/src/sqlbuild/presentation/models.py index 18264c6368..9dc8455ead 100644 --- a/src/sqlbuild/presentation/models.py +++ b/src/sqlbuild/presentation/models.py @@ -13,47 +13,3 @@ class DisplayOptions: max_entries_per_section: int | None = DEFAULT_MAX_DISPLAY_ENTRIES overflow_flag: str = "--verbose" - - -@dataclass(frozen=True) -class TextStyle: - """One ANSI style role in the CLI theme.""" - - prefix: str - suffix: str = "\033[0m" - - def apply(self, *, text: str, use_color: bool) -> str: - """Apply this style when color is enabled.""" - - if not use_color or not self.prefix or not text: - return text - return f"{self.prefix}{text}{self.suffix}" - - -@dataclass(frozen=True) -class CliTheme: - """Semantic style roles for human CLI output.""" - - title: TextStyle - section: TextStyle - label: TextStyle - value: TextStyle - accent: TextStyle - accent_strong: TextStyle - plan_section: TextStyle - object_name: TextStyle - command: TextStyle - success: TextStyle - success_strong: TextStyle - warning: TextStyle - warning_strong: TextStyle - error: TextStyle - error_strong: TextStyle - error_muted: TextStyle - log_label: TextStyle - skipped: TextStyle - muted: TextStyle - dbt_section: TextStyle - dbt_label: TextStyle - dbt_object_name: TextStyle - dbt_execution_label: TextStyle diff --git a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py index 32a8c500c5..7b7d3ad7e4 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py @@ -3,7 +3,6 @@ from __future__ import annotations from concurrent.futures import Executor, Future, ThreadPoolExecutor -from contextvars import Token from pathlib import Path from threading import Event @@ -11,6 +10,7 @@ build_lint_expansion_context, early_lint_expansions, finish_rule_run, + lint_stop_scope, plan_rule_run, run_lint, run_planned_sql_rules, @@ -25,7 +25,6 @@ from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.lint.classes.stop_context import LintStopContext from sqlbuild.lint.models import LintConfig, LintRunResult from sqlbuild.rule_engine._helpers.engine.custom_dispatch import evaluate_native, start_custom_rules from sqlbuild.rule_engine.models import ( @@ -133,8 +132,7 @@ def _prepare_sql_lint( compiled_expansions: dict[Path, CompiledSqlExpansion], stop: Event, ) -> PreparedSqlLintResult: - stop_token: Token[Event | None] = LintStopContext.active.set(stop) - try: + with lint_stop_scope(stop): context: SqlExpansionContext = build_lint_expansion_context( project_dir=project_dir, discovered_inputs=discovered_inputs, @@ -147,6 +145,4 @@ def _prepare_sql_lint( compiled_expansions=compiled_expansions, expansion_context=context, ) - finally: - LintStopContext.active.reset(stop_token) return PreparedSqlLintResult(result=result, context=context) diff --git a/src/sqlbuild/runtime/contracts/main/__init__.py b/src/sqlbuild/runtime/contracts/main/__init__.py deleted file mode 100644 index e5537acbae..0000000000 --- a/src/sqlbuild/runtime/contracts/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Runtime contract operations.""" diff --git a/src/sqlbuild/runtime/contracts/main/open_connection.py b/src/sqlbuild/runtime/contracts/main/open_connection.py deleted file mode 100644 index 8c92accf53..0000000000 --- a/src/sqlbuild/runtime/contracts/main/open_connection.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Callback-aware adapter connection opening.""" - -from __future__ import annotations - -import time -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.runtime.contracts.models import ConnectionHooks - - -def open_connection_with_hooks( - *, - adapter: BaseAdapter, - connection_config: dict[str, object], - hooks: ConnectionHooks, -) -> Any: - """Open one adapter connection and report its lifecycle through hooks.""" - - if hooks.on_connection_start is not None: - hooks.on_connection_start(1) - started_at: float = time.monotonic() - try: - connection: Any = adapter.connect(connection_config) - except Exception: - if hooks.on_connection_error is not None: - hooks.on_connection_error(1, elapsed_seconds=time.monotonic() - started_at) - raise - if hooks.on_connection_complete is not None: - hooks.on_connection_complete(1, elapsed_seconds=time.monotonic() - started_at) - return connection diff --git a/src/sqlbuild/sql_values/constants.py b/src/sqlbuild/sql_values/constants.py index 472af39e48..6fee16de1b 100644 --- a/src/sqlbuild/sql_values/constants.py +++ b/src/sqlbuild/sql_values/constants.py @@ -1,8 +1,5 @@ """Constants for typed SQL value validation.""" -MIN_SIGNED_64_BIT_INTEGER: int = -(2**63) -MAX_SIGNED_64_BIT_INTEGER: int = 2**63 - 1 DEFAULT_MAX_SQL_VALUE_DEPTH: int = 32 DEFAULT_MAX_SQL_VALUE_ELEMENTS: int = 10_000 DEFAULT_MAX_SQL_VALUE_SIZE: int = 1_000_000 -MISSING_SQL_VALUE: object = object() diff --git a/tests/conftest.py b/tests/conftest.py index 2427d418f2..d1b10120dc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,11 +6,13 @@ import pytest -from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR, REUSE_DISABLE_VALUE +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, +) def pytest_configure(config: pytest.Config) -> None: """Exercise full compiles by default; compile reuse tests opt back in explicitly.""" del config - os.environ.setdefault(REUSE_DISABLE_ENV_VAR, REUSE_DISABLE_VALUE) + os.environ.setdefault(REUSE_DISABLE_ENV_VAR, "1") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py index 49fe59ce12..23b64d7332 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory +from sqlbuild._native import compiler_cache_directory from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( RuntimeArtifactPreservationBuildE2ETestCase, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py index 0ec74c34da..542142b45c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py @@ -7,7 +7,9 @@ from pathlib import Path from typing import NamedTuple, cast -from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, +) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import run_installed_sqb _TIMING: re.Pattern[str] = re.compile(r" \(\d+\.\d+s\)") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index ad3bd2d6d9..15334996b8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -30,9 +30,9 @@ import pytest import sqlbuild._native as native_module +import sqlbuild._native as plan_artifacts_module import sqlbuild.adapter.type_system._helpers.type_normalization as type_normalization import sqlbuild.cli.compile_reuse._helpers.provider_settings as provider_settings -import sqlbuild.cli.compile_reuse.constants as reuse_constants import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module import sqlbuild.compiler.frontier.main.compiled_code_identity as compiled_code_identity_module import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class @@ -41,17 +41,10 @@ semantic_compile_fingerprint, ) from scripts.compiler_differential.constants import FAILURE_BASE_FILES +from sqlbuild._native import compiler_cache_directory from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.types import TypeDialect -from sqlbuild.cli.compile_reuse.constants import ( - EXCLUDED_ROOT_DIRECTORIES, - NATIVE_REUSE_DIRECTORY_NAME, - NATIVE_REUSE_SUFFIX, - RETIRED_REUSE_DIRECTORY_NAME, - REUSE_DISABLE_ENV_VAR, - REUSE_LOGGER_NAME, -) from sqlbuild.compiler.analysis_session.constants import NATIVE_ANALYSIS_STORE_FILE_NAME from sqlbuild.compiler.compile.constants import ( RETIRED_FACT_CACHE_DIRECTORY_NAME, @@ -59,15 +52,14 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.contracts.models import ContractValidationResult +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, +) from sqlbuild.compiler.frontier.constants import ( COMPILER_CACHE_DIRECTORY_NAME, RETIRED_CACHE_NAMESPACE_SUFFIXES, ) -from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME -from sqlbuild.compiler.planner.main.execution import ( - _plan_compiled_sql_test_artifacts as plan_artifacts_module, -) from sqlbuild.observability import EventDispatcher, LifecycleEvent from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( FreshProcessCompileCachePerformanceGuardTestCase, @@ -85,6 +77,10 @@ contract_statuses, ) +NATIVE_REUSE_DIRECTORY_NAME: str = "project-reuse-v2" +NATIVE_REUSE_SUFFIX: str = ".store" +RETIRED_REUSE_DIRECTORY_NAME: str = "project-reuse-v1" +REUSE_LOGGER_NAME: str = "sqlbuild.compile.reuse" _DBT_SHAPED_SQL_SIZE_PROFILE: tuple[tuple[float, int], ...] = ( (0.50, 1_800), (0.75, 4_500), @@ -165,6 +161,14 @@ class SemanticCompileBenchmarkResult(NamedTuple): warm: CompileBenchmarkMeasurement +_RETAINED_BENCHMARK_PAYLOAD_KEYS: tuple[str, ...] = ( + "compile_timings", + "has_errors", + "diagnostics", + "summary", +) + + class FreshProcessCompileBenchmarkResult(NamedTuple): elapsed_seconds: float peak_rss_bytes: int @@ -664,6 +668,7 @@ def _run_fresh_process_compile_benchmark( label: str, expected_max_wall_seconds: float, compile_args: tuple[str, ...], + retained_payload_keys: tuple[str, ...] = _RETAINED_BENCHMARK_PAYLOAD_KEYS, ) -> FreshProcessCompileBenchmarkResult: run: _TimedSqbRun = _run_timed_sqb( project_dir=project_dir, @@ -681,7 +686,7 @@ def _run_fresh_process_compile_benchmark( elapsed_seconds=run.elapsed_seconds, peak_rss_bytes=run.peak_rss_bytes, semantic_fingerprint=semantic_fingerprint, - payload=payload, + payload={key: payload.get(key) for key in retained_payload_keys}, cpu_seconds=run.cpu_seconds, major_page_faults=run.major_page_faults, minor_page_faults=run.minor_page_faults, @@ -3397,7 +3402,7 @@ def record_digested_files(*, monkeypatch: pytest.MonkeyPatch) -> list[int]: def settle_racy_window(*, monkeypatch: pytest.MonkeyPatch) -> None: """Treat files written before a compile starts as settled instead of waiting two seconds.""" - monkeypatch.setattr(reuse_constants, "RACY_WINDOW_NS", 0) + monkeypatch.setattr(native_module, "COMPILE_REUSE_RACY_WINDOW_NS", 0) def write_large_file(*, project_dir: Path, relative_path: str, size_bytes: int) -> Path: @@ -4221,7 +4226,9 @@ def ignore_project_changes(monkeypatch: pytest.MonkeyPatch) -> None: """Make compile reuse stop watching the models folder, so a model edit goes unseen.""" monkeypatch.setattr( - reuse_constants, "EXCLUDED_ROOT_DIRECTORIES", EXCLUDED_ROOT_DIRECTORIES | {"models"} + native_module, + "COMPILE_REUSE_EXCLUDED_ROOT_DIRECTORIES", + native_module.COMPILE_REUSE_EXCLUDED_ROOT_DIRECTORIES | {"models"}, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py index 57a431ed34..22e2a5d2bb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py @@ -6,7 +6,9 @@ from pathlib import Path from typing import Any, NamedTuple, cast -from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, +) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import run_installed_sqb diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index 8cdfdeb7ca..b30fcfe4c8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -9,10 +9,9 @@ from textwrap import dedent from types import MappingProxyType -from sqlbuild._native import write_fingerprint +from sqlbuild._native import write_fingerprint, write_source_freshness_records from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.source_freshness.main.write import write_source_freshness_records from sqlbuild.compiler.source_freshness.models import ( SourceFreshnessRecord, SourceFreshnessRenderers, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py index eb59249a10..d3e3787872 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py @@ -14,7 +14,9 @@ from pathlib import Path from typing import IO, NamedTuple, cast -from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, +) from sqlbuild.compiler.planner.types import ScenarioArtifactKind from sqlbuild.executor.scenario._helpers.snapshots.core import ( build_scenario_snapshot_input_fingerprint, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py index 55fa9e2633..3bcbe1fb76 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py @@ -8,7 +8,7 @@ from pathlib import Path from time import perf_counter -from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory +from sqlbuild._native import compiler_cache_directory from sqlbuild.compiler.scopes.constants import SCOPE_CACHE_DIRECTORY_NAME, SCOPE_CACHE_FILENAME from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import REPO_ROOT diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/test_cost.py b/tests/integration/src/sqlbuild/adapters/snowflake/test_cost.py index 27698afa10..8a93a03435 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/test_cost.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/test_cost.py @@ -7,9 +7,8 @@ import pytest +from sqlbuild._native import CostContext, collect_snowflake_cost from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.cost.classes.cost_context import CostContext -from sqlbuild.cost.main.collection import collect_snowflake_cost from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostStatus from tests.integration.src.sqlbuild.adapters.snowflake._test_types import ( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index d4cc7e7c3b..8e0a72a9d2 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -939,6 +939,15 @@ class WarmTargetEditTestCase: expected_exit_code: int = 0 +@dataclass(frozen=True) +class WarmTargetProjectConfigEditTestCase: + description: str + edited_project_config: str + expected_edit_changed: tuple[str, ...] = () + expected_after_edit_changed: tuple[str, ...] = () + expected_exit_codes: tuple[int, int] = (0, 0) + + @dataclass(frozen=True) class RuleExceptionPolicyTestCase: """One `[rules] allow_exceptions` value with one escape hatch, compiled through the CLI.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py index e24bb68e25..6dab1de89a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py @@ -11,21 +11,19 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import PreparedCompileArtifacts +from sqlbuild._native import PreparedCompileArtifacts, compiler_cache_directory from sqlbuild._native import run_cli as main -from sqlbuild.cli.compile_reuse import constants as reuse_constants_module from sqlbuild.cli.compile_reuse._helpers import provider_settings as provider_settings_module from sqlbuild.cli.compile_reuse._helpers import runtime_identity as runtime_identity_module from sqlbuild.cli.compile_reuse._helpers.runtime_identity import runtime_identity -from sqlbuild.cli.compile_reuse.constants import ( - NATIVE_REUSE_DIRECTORY_NAME, - NATIVE_REUSE_SUFFIX, - REUSE_DISABLE_ENV_VAR, +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, ) from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_DIR_ENV_VAR -from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory STAGING_PREFIX: str = ".sqlbuild-staging-" +NATIVE_REUSE_DIRECTORY_NAME: str = "project-reuse-v2" +NATIVE_REUSE_SUFFIX: str = ".store" class _StagingEveryProject(PreparedCompileArtifacts): @@ -110,7 +108,7 @@ def prepare_output_compiles(*, monkeypatch: pytest.MonkeyPatch, reuse_disabled: monkeypatch.setenv(REUSE_DISABLE_ENV_VAR, reuse_disabled) monkeypatch.setattr(native_module, "PreparedCompileArtifacts", _StagingEveryProject) - monkeypatch.setattr(reuse_constants_module, "RACY_WINDOW_NS", 0) + monkeypatch.setattr(native_module, "COMPILE_REUSE_RACY_WINDOW_NS", 0) def main_compile_json(*, project_dir: Path) -> int: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 32090718a4..ab7a1ba08f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -14,6 +14,7 @@ import pytest import sqlbuild._native as native_module +import sqlbuild._native as plan_artifacts_module from sqlbuild._native import PlannedStaticSqlTests, PreparedCompileArtifacts from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.exceptions import AdapterUserError @@ -28,9 +29,6 @@ from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_SUFFIX from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.planner.exceptions import NativeSqlTestPlanningError -from sqlbuild.compiler.planner.main.execution import ( - _plan_compiled_sql_test_artifacts as plan_artifacts_module, -) from sqlbuild.compiler.planner.models import NativeSqlTestArtifact from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( RepeatedJsonParseTestCase, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_warm_target.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_warm_target.py index 6af6a5fbb1..45dc6dac01 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_warm_target.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_warm_target.py @@ -17,6 +17,7 @@ ) from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( WarmTargetEditTestCase, + WarmTargetProjectConfigEditTestCase, WarmTargetTestCase, ) @@ -96,5 +97,44 @@ def test_given_one_edited_model_when_compiling_warm_then_rewrites_only_its_artif ) +@pytest.mark.parametrize( + "test_case", + [ + WarmTargetProjectConfigEditTestCase( + "unused_var_added", + edited_project_config=('name = "shop"\nadapter = "duckdb"\n\n[vars]\nrevision = "1"\n'), + ) + ], + ids=lambda case: case.description, +) +def test_given_project_config_edit_when_compiling_twice_then_rewrites_no_unchanged_artifact( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + test_case: WarmTargetProjectConfigEditTestCase, +) -> None: + write_warm_target_project(project_dir=tmp_path) + _ = compile_project(project_dir=tmp_path, compile_args=()) + compiled_dir: Path = tmp_path / "target" / "compiled" + stamp_artifacts( + artifacts=compiled_artifacts(compiled_dir=compiled_dir), mtime_ns=_UNCHANGED_MTIME_NS + ) + (tmp_path / "sqlbuild_project.toml").write_text( + test_case.edited_project_config, encoding="utf-8" + ) + + edit_exit_code: int = compile_project(project_dir=tmp_path, compile_args=()) + edit_changed: tuple[str, ...] = changed_artifacts( + compiled_dir=compiled_dir, mtime_ns=_UNCHANGED_MTIME_NS + ) + after_edit_exit_code: int = compile_project(project_dir=tmp_path, compile_args=()) + _ = capsys.readouterr() + + assert (edit_exit_code, after_edit_exit_code) == test_case.expected_exit_codes + assert edit_changed == test_case.expected_edit_changed + assert changed_artifacts(compiled_dir=compiled_dir, mtime_ns=_UNCHANGED_MTIME_NS) == ( + test_case.expected_after_edit_changed + ) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 289af32bd6..158d3ee9d7 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -15,12 +15,11 @@ import sqlbuild.compiler.compile._helpers.render.user_macros as macros_module import sqlbuild.rule_engine._helpers.host.custom_host_pool as custom_host_pool_module import sqlbuild.rule_engine._helpers.run.rule_threads as rule_threads_module -from sqlbuild._native import rules_bulk_cache_path +from sqlbuild._native import active_lint_stop, rules_bulk_cache_path from sqlbuild._native import run_cli as main from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, DiscoveredProjectInputs from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.lint.classes.stop_context import LintStopContext from sqlbuild.lint.models import LintRunResult from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( CustomHostSplitIntegrationTestCase, @@ -1037,7 +1036,7 @@ def test_given_graph_error_when_compiling_then_early_lint_is_stopped_instead_of_ def stop_awaiting_run_lint(**kwargs: Any) -> LintRunResult: lint_started.set() - stop: Event | None = LintStopContext.active.get() + stop: Event | None = active_lint_stop() stop_observed.append(stop is not None and stop.wait(timeout=_EARLY_LINT_STOP_TIMEOUT_S)) return original_run_lint(**kwargs) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py index c170be2c02..5a2ae9f03a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -11,7 +11,9 @@ import pytest -from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, +) from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( ReusedCompileImportFootprintTestCase, StartupImportFootprintTestCase, diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index dc5bba1d5b..9e25be6485 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -7,7 +7,9 @@ from pathlib import Path import sqlbuild -from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, +) type LauncherRun = tuple[int, bytes, bytes] diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index cc5bc10b8b..9ad5adf381 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -16,6 +16,7 @@ import sqlbuild._native as native_module from sqlbuild._native import ( analyze_native_model_sql, + build_analysis_cache_context, build_compile_inputs, cursor_intrinsics_analysis_sql, discover_project_inputs, @@ -36,7 +37,6 @@ NativePivotTables, ) from sqlbuild.compiler.analysis_session.types import ProofRow -from sqlbuild.compiler.compile._helpers.analysis.cache import build_analysis_cache_context from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( AnalysisCacheContext, diff --git a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py index 144f892d98..8504effbe6 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py @@ -15,7 +15,6 @@ NativeModelConfigSession, ) from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile -from sqlbuild.compiler.model_config.main._native_config_error import native_config_error from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import ProjectConfig @@ -83,10 +82,7 @@ def validation_outcome(*, values: dict[str, object]) -> object: ) return [ _ACCEPTED, - *( - raised_error_shape(native_config_error(error=error, values=values)) - for error in filter(None, (outcome,)) - ), + *(raised_error_shape(error.exception(values=values)) for error in filter(None, (outcome,))), ][-1] diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index 047e73d94f..66bd89520c 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -22,7 +22,6 @@ from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.runtime.contracts.main import open_connection as compile_pipeline from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( AppendCursorPipelineIntegrationTestCase, CompileProgressIntegrationTestCase, @@ -141,13 +140,14 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i 'MODEL (description "Orders");\nSELECT order_id FROM orders LIMIT 1\n', encoding="utf-8", ) - open_connection: Mock = Mock() - monkeypatch.setattr(compile_pipeline, "open_connection_with_hooks", open_connection) + adapter: DuckDbAdapter = DuckDbAdapter() + connect: Mock = Mock() + monkeypatch.setattr(adapter, "connect", connect) with pytest.raises(RulesError, match=test_case.expected_error_pattern): - _ = run_compile_pipeline_for_project(project_dir=tmp_path, adapter=DuckDbAdapter()) + _ = run_compile_pipeline_for_project(project_dir=tmp_path, adapter=adapter) - assert open_connection.call_count == test_case.expected_connection_calls + assert connect.call_count == test_case.expected_connection_calls @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py index cafffb3ff6..ab108f5ff5 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py @@ -7,11 +7,11 @@ import pytest from sqlbuild._native import run_cli as main -from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( IdentifierBindingCase, NativeCatalogCase, ) +from tests.unit.src.sqlbuild.compiler.sql_analysis.main.helpers import normalize_analysis_sql @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index 65e2308310..e5e227e68a 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -6,7 +6,12 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs +from sqlbuild._native import ( + assemble_project, + build_compile_inputs, + discover_project_inputs, + resolve_model_sql, +) from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -22,7 +27,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_model_sql from sqlbuild.compiler.planner.models import ( BackfillResult, CursorOverridePair, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py index 13cd341c29..1b1ca53e1b 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import resolve_source_references from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.planner._helpers.resolve.sources import resolve_source_references from sqlbuild.spec.contracts.models import SourceColumnEntry, SourceEntry from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( AdapterOverrideTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index 3a910007b9..ea7d115776 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -9,12 +9,12 @@ build_compiled_project, build_relation_fixture_completion, build_relation_fixture_context, + build_selected_test_entries, discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner._helpers.output.plan_output import build_selected_test_entries from sqlbuild.compiler.planner.models import ( RelationFixtureCompletion, RelationFixturePlanningContext, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index bc10eaabad..6df338e9b3 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -14,6 +14,7 @@ from sqlbuild._native import ( build_compiled_function_fingerprint_sql, + build_execution_plan, compiled_model_record, compute_query_hash, discover_project_inputs, @@ -52,7 +53,6 @@ ) from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult -from sqlbuild.compiler.planner.main.execution.execution import build_execution_plan from sqlbuild.compiler.planner.models import ( DeferralInputs, PlannerOverrides, diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py index 4c3a914582..8b1e1cb324 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -18,9 +18,9 @@ build_compile_inputs, discover_project_inputs, inputs_with, + record_compile_input_reads, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS from sqlbuild.compiler.compile.models import ( CompileAdapterContext, ) @@ -168,7 +168,7 @@ def recorded_assembly( ) -> tuple[object, tuple[tuple[str, ...], bool]]: """Assemble `inputs`; return its assembly view and the input reads recorded.""" - with COMPILE_INPUT_READS.recording() as reads: + with record_compile_input_reads() as reads: project: CompiledProjectView = assemble_with(inputs=inputs, monkeypatch=monkeypatch) return assembly_view(project), (reads.environment_names, reads.read_run_id) diff --git a/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py b/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py index c5de3b099e..5e7d43a15e 100644 --- a/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py +++ b/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py @@ -6,11 +6,10 @@ import pytest +from sqlbuild._native import read_latest_source_freshness, write_source_freshness_records from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME -from sqlbuild.compiler.source_freshness.main.read import read_latest_source_freshness -from sqlbuild.compiler.source_freshness.main.write import write_source_freshness_records from sqlbuild.compiler.source_freshness.models import ( SourceFreshnessIdentity, SourceFreshnessRecord, diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py index 272150769b..ed4d0986b8 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py @@ -9,8 +9,8 @@ import pytest +from sqlbuild._native import CostContext from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus from tests.integration.src.sqlbuild.executor.build.concurrent._test_types import ( diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py index e89b3c5ddc..b587e45dff 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py @@ -7,10 +7,10 @@ import pytest +from sqlbuild._native import compute_cursor_bounds from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.planner._helpers.resolve.cursor import compute_cursor_bounds from sqlbuild.compiler.planner.models import ( BackfillResult, CursorBounds, diff --git a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py index 491d20a309..3a9794b84f 100644 --- a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py +++ b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py @@ -4,7 +4,6 @@ from fensu import RuleCase, RuleResult, evaluate_rule from scripts.fensu_policy.rules.canonical_operations import ( - color_capability_entry, dbt_graph_projection, dbt_reference_resolution, selector_marker_parsing, @@ -15,45 +14,6 @@ from tests.unit.scripts.fensu_policy.rules._test_types import CustomRuleTestCase -@pytest.mark.parametrize( - "test_case", - [ - CustomRuleTestCase( - description="raw terminal capability import faults", - path="src/sqlbuild/example/main/build.py", - source=( - "from sqlbuild.presentation._helpers.terminal_capabilities import terminal_columns\n" - ), - expected_fault_count=1, - ), - CustomRuleTestCase( - description="public terminal capability import passes", - path="src/sqlbuild/example/main/build.py", - source="from sqlbuild.presentation.main.terminal_columns import terminal_columns\n", - expected_fault_count=0, - ), - ], - ids=lambda case: case.description, -) -def test_given_import_when_checking_color_entry_then_matches_contract( - test_case: CustomRuleTestCase, -) -> None: - result: RuleResult = evaluate_rule( - rule=color_capability_entry, - test_case=RuleCase( - description=test_case.description, - source=test_case.source, - expected_fault_count=test_case.expected_fault_count, - path=test_case.path, - scope=test_case.scope, - scope_root=test_case.scope_root, - files=test_case.files, - ), - ) - - assert result.fault_count == test_case.expected_fault_count - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py index 468c2c0081..5f6df47ca4 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -114,6 +114,7 @@ def test_given_native_source_when_checking_module_paths_then_reports_errors( "sqlbuild.cli.commands.classes.prepared_compile_artifacts", "sqlbuild.cli.observability", "sqlbuild.compile", + "sqlbuild.compile.reuse", "sqlbuild.cost", "sqlbuild.executor.runtime", "sqlbuild.fingerprints", diff --git a/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py b/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py index f762fe4391..1375a23885 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py +++ b/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py @@ -23,7 +23,6 @@ "adapters/snowflake/classes/snowflake_adapter.py", "adapters/sqlserver/classes/sqlserver_adapter.py", "adapters/sqlserver/constants.py", - "cost/_helpers/snowflake.py", "cost/models.py", } ) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/classes/snowflake_connection/test_snowflake_connection.py b/tests/unit/src/sqlbuild/adapters/snowflake/classes/snowflake_connection/test_snowflake_connection.py index 39ce775b47..6e73d0ec60 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/classes/snowflake_connection/test_snowflake_connection.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/classes/snowflake_connection/test_snowflake_connection.py @@ -8,10 +8,9 @@ import pytest +from sqlbuild._native import COST_TELEMETRY_HEALTH, CostContext from sqlbuild.adapters.snowflake.classes.snowflake_connection import _SnowflakeConnection from sqlbuild.adapters.snowflake.classes.snowflake_cursor import _SnowflakeCursor -from sqlbuild.cost.classes.cost_context import CostContext -from sqlbuild.cost.constants import COST_TELEMETRY_HEALTH from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.observability import ( EventDispatcher, @@ -512,13 +511,9 @@ def test_given_ledger_write_failure_when_executing_then_statement_result_is_unch ) -> None: cursor: _Cursor = _Cursor() connection: _SnowflakeConnection = _SnowflakeConnection(_RawConnection(cursor)) - ledger_path: Path = tmp_path / "statements.jsonl" - - def fail_append(**kwargs: Any) -> None: - del kwargs - raise OSError("disk unavailable") - - monkeypatch.setattr("sqlbuild.cost._helpers.ledger._append_statement", fail_append) + blocker: Path = tmp_path / "blocker" + blocker.write_text("not a directory", encoding="utf-8") + ledger_path: Path = blocker / "statements.jsonl" with CostContext.scope( run_id="ledger-failure-run", resource_type="model", @@ -531,7 +526,9 @@ def fail_append(**kwargs: Any) -> None: assert cursor.sfqid == test_case.expected_query_id assert test_case.expected_status == "success" assert not ledger_path.exists() - assert COST_TELEMETRY_HEALTH.consume_ledger_failure(run_id="ledger-failure-run") == "OSError" + assert COST_TELEMETRY_HEALTH.consume_ledger_failure(run_id="ledger-failure-run") == ( + "FileExistsError" + ) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py index aec230981d..9249d6654f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py @@ -4,13 +4,6 @@ from sqlbuild.adapter.contract.models import RetentionRequest from sqlbuild.adapter.contract.types import RetentionScope -from sqlbuild.cli.commands.constants import ( - EXECUTION_LIMIT_BUILD_NOTE, - MISSING_ORIGIN_BUILD_HELP, - RETENTION_DECREASE_BUILD_HELP, - SNAPSHOT_FULL_REFRESH_BUILD_HELP, - TABLE_TYPE_DOWNGRADE_BUILD_HELP, -) from sqlbuild.cli.commands.models import PlanSafetyGates from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType @@ -123,3 +116,18 @@ def build_policy_gates( retention_decrease_help=RETENTION_DECREASE_BUILD_HELP, execution_limit_note=EXECUTION_LIMIT_BUILD_NOTE, ) + + +EXECUTION_LIMIT_BUILD_NOTE: str = "No warehouse changes were made." +MISSING_ORIGIN_BUILD_HELP: str = ( + "Pass --allow-missing-migration-origin to confirm in non-interactive runs." +) +RETENTION_DECREASE_BUILD_HELP: str = ( + "Pass --allow-retention-decrease to confirm in non-interactive runs." +) +SNAPSHOT_FULL_REFRESH_BUILD_HELP: str = ( + "Pass --allow-snapshot-full-refresh to confirm in non-interactive runs." +) +TABLE_TYPE_DOWNGRADE_BUILD_HELP: str = ( + "Pass --allow-table-type-downgrade to confirm in non-interactive runs." +) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py index 9084ca978e..b0cc3de61b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py @@ -1,12 +1,14 @@ import pytest from sqlbuild._native import enforce_model_execution_limit -from sqlbuild.cli.commands.constants import EXECUTION_LIMIT_BUILD_NOTE from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.spec.contracts.models import ExecutionLimitsConfig from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( ModelExecutionLimitTestCase, ) +from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning.helpers import ( + EXECUTION_LIMIT_BUILD_NOTE, +) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py index b7c2fc1f1d..25acc186fd 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py @@ -11,7 +11,6 @@ from sqlbuild._native import read_cost_run, refresh_pending_cost_run, write_cost_run from sqlbuild.cli.commands.models import CostCommandRequest -from sqlbuild.cost.constants import LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX from sqlbuild.cost.models import CostRunRecord, RunCostSummary from sqlbuild.cost.types import CostCapability, CostStatus from tests.unit.src.sqlbuild.cli.commands._helpers.cost._test_types import ( @@ -249,7 +248,7 @@ def test_given_persisted_ledger_failure_when_refresh_has_no_coverage_then_failur ) -> None: now: datetime = datetime(2026, 8, 23, tzinfo=UTC) record: CostRunRecord = build_cost_run_record(run_id="run-ledger-failure", completed_at=now) - failure_limitation: str = f"{LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX}OSError)." + failure_limitation: str = "Statement-ledger persistence failed (OSError)." persisted_failure: RunCostSummary = RunCostSummary( status=CostStatus.PARTIAL, usd_per_credit=record.cost.usd_per_credit, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py index dda7c71329..5054627b09 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py @@ -3,6 +3,7 @@ from datetime import UTC, datetime from pathlib import Path +from sqlbuild._native import CostContext, build_source_freshness_create_table_sql from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -12,14 +13,10 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.models import ChainStep, ModelPlanEntry, SqlTestPlanEntry from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason -from sqlbuild.compiler.source_freshness.main.create_table_sql import ( - build_create_table_sql as build_source_freshness_create_table_sql, -) from sqlbuild.compiler.source_freshness.models import ( SourceFreshnessIdentity, SourceFreshnessRecord, ) -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py index 31d2463627..b7cfa9bd13 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py @@ -89,13 +89,6 @@ class WarehouseFlagRejectedTestCase: expected_error_fragment: str = "unrecognized arguments: --warehouse" -@dataclass(frozen=True) -class CommandWarehouseClassificationTestCase: - description: str - command: CliCommand - expected_classified: bool = True - - @dataclass(frozen=True) class CommandWarehouseGroupTestCase: description: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py index ef9873a880..d703c13970 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py @@ -8,11 +8,11 @@ from sqlbuild._native import ( command_warehouse_scope, + current_command_warehouse_scope, describe_connection_warehouse, parse_cli_invocation, resolve_project_connection_config, ) -from sqlbuild.cli.commands.constants import COMMAND_WAREHOUSE_GROUPS from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.types import CliCommand from sqlbuild.cli.entry.models import ParsedCliInvocation, ResolvedWarehouse @@ -20,7 +20,6 @@ from sqlbuild.spec.contracts.models import LocalTargetConfig, TargetWarehousesConfig from sqlbuild.spec.contracts.types import WarehouseGroup from tests.unit.src.sqlbuild.cli.commands._helpers.runtime._test_types import ( - CommandWarehouseClassificationTestCase, CommandWarehouseErrorTestCase, CommandWarehouseGroupTestCase, CommandWarehouseResolutionTestCase, @@ -263,20 +262,6 @@ def test_given_adapter_without_warehouses_when_cli_warehouse_is_given_then_it_fa assert test_case.expected_message_fragment in error.value.message -@pytest.mark.parametrize( - "test_case", - [ - CommandWarehouseClassificationTestCase(description=command.value, command=command) - for command in CliCommand - ], - ids=lambda case: case.description, -) -def test_given_cli_command_when_checking_warehouse_groups_then_it_is_classified( - test_case: CommandWarehouseClassificationTestCase, -) -> None: - assert (test_case.command in COMMAND_WAREHOUSE_GROUPS) == test_case.expected_classified - - @pytest.mark.parametrize( "test_case", [ @@ -380,7 +365,8 @@ def test_given_command_without_group_when_parsing_warehouse_flag_then_it_is_reje def test_given_cli_command_when_resolving_group_then_matches_its_workload( test_case: CommandWarehouseGroupTestCase, ) -> None: - assert COMMAND_WAREHOUSE_GROUPS[test_case.command] is test_case.expected_group + with command_warehouse_scope(command=test_case.command, cli_warehouse=None): + assert current_command_warehouse_scope().group is test_case.expected_group if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py index aa3eebe902..7360cdc360 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py @@ -6,6 +6,7 @@ import pytest +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.freshness.source_freshness import ( append_eligible_direct_source_freshness_records, @@ -15,7 +16,6 @@ DirectSourceFreshnessPlanningResult, DirectSourceFreshnessPropagationResult, ) -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 94e79a7794..bd858d132b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -11,14 +11,12 @@ import pytest import sqlbuild._native as discovery_entry +import sqlbuild._native as plan_artifacts_module from sqlbuild._native import run_compile from sqlbuild.adapter.discovery.main import resolve_adapter as runtime_adapters from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.entry.models import CompileCommandRequest from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.compiler.planner.main.execution import ( - _plan_compiled_sql_test_artifacts as plan_artifacts_module, -) from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileColumnContractModeTestCase, CompileCommandTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 047dd8951e..118615bd48 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -8,6 +8,8 @@ import pytest import sqlbuild._native as _native +import sqlbuild._native as model_chain_module +import sqlbuild._native as plan_artifacts_module from sqlbuild._native import ( build_manifest, compiled_model_record, @@ -26,10 +28,6 @@ CompiledResourceType, DiagnosticPhase, ) -from sqlbuild.compiler.planner.main.execution import ( - _plan_compiled_sql_test_artifacts as plan_artifacts_module, -) -from sqlbuild.compiler.planner.main.execution import sql_test_model_chain as model_chain_module from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/conftest.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/conftest.py index d48b040d55..6b1be16d45 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/conftest.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/conftest.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.cli.commands.constants import SQLBUILD_CONCURRENCY_ENV_VAR +SQLBUILD_CONCURRENCY_ENV_VAR: str = "SQLBUILD_CONCURRENCY" @pytest.fixture(autouse=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_connection_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_connection_progress.py index f39b25c692..d2daea412f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_connection_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_connection_progress.py @@ -7,9 +7,9 @@ import pytest from sqlbuild._native import ( + CliStyle, ConnectionProgressReporter, ) -from sqlbuild.presentation.classes.cli_style import CliStyle from tests.unit.src.sqlbuild.cli.commands.shared._helpers._test_types import ( ConnectionProgressTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_planning_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_planning_progress.py index 0813bd636d..059fdfa5e3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_planning_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_planning_progress.py @@ -7,9 +7,9 @@ import pytest from sqlbuild._native import ( + CliStyle, PlanningProgressReporter, ) -from sqlbuild.presentation.classes.cli_style import CliStyle from tests.unit.src.sqlbuild.cli.commands.shared._helpers._test_types import ( PlanningFinishTestCase, PlanningProgressTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py index 77e77250e7..535810c1d7 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py @@ -3,7 +3,6 @@ from __future__ import annotations import time -from functools import partial from io import StringIO import pytest @@ -36,7 +35,6 @@ from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.executor.testing.types import SqlTestOutcome -from sqlbuild.presentation.main import terminal_columns as terminal_columns_module from sqlbuild.runtime.contracts.types import ExecutionResourceKind from tests.unit.src.sqlbuild.cli.commands.shared._helpers._test_types import ( BuildFooterTestCase, @@ -695,11 +693,7 @@ def test_given_failed_top_level_node_when_reporting_progress_then_writes_error_d ) -> None: stream: StringIO = StringIO() monkeypatch.setattr("sys.stdout", stream) - monkeypatch.setattr( - terminal_columns_module, - "terminal_columns", - partial(int, test_case.terminal_width), - ) + monkeypatch.setenv("COLUMNS", str(test_case.terminal_width)) callbacks: BuildProgressCallbacks = BuildProgressCallbacks( plan=PlanOutput(), use_color=test_case.use_color, diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py b/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py index b322238284..356455b32f 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py @@ -9,7 +9,9 @@ from pathlib import Path from typing import cast -from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, +) def compile_in_fresh_process( diff --git a/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py b/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py index 8206664aec..856756ee1c 100644 --- a/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py +++ b/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py @@ -8,7 +8,7 @@ import pytest -from sqlbuild._native import NativeProgressProjector +from sqlbuild._native import NativeProgressProjector, shared_transient_line_coordinator from sqlbuild.observability import ( EventDispatcher, LifecycleEvent, @@ -18,7 +18,6 @@ invocation_scope, run_scope, ) -from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator from sqlbuild.runtime.observability.classes.operation_lifecycle import publish_retry_scheduled from tests.unit.src.sqlbuild.cli.progress.classes._test_types import ( CursorCleanupCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 230ed782a7..b81b5a40bf 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -82,22 +82,6 @@ class RetentionValidationValidTestCase: expected_unmanaged: bool -@dataclass(frozen=True) -class FindMatchingPathDefaultTestCase: - description: str - model_relative_path: str - path_defaults: dict[str, dict[str, object]] - expected_path_default: str | None - - -@dataclass(frozen=True) -class FindMatchingPathDefaultErrorTestCase: - description: str - model_relative_path: str - path_defaults: dict[str, dict[str, object]] - expected_error_fragment: str - - @dataclass(frozen=True) class LoadProjectMacrosTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py index 15e844b23e..2c34d0dfde 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py @@ -16,7 +16,6 @@ NativeModelConfigSession, ) from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.model_config.main._native_config_error import native_config_error from sqlbuild.spec.contracts.models import ( DefaultsConfig, MaterializationDefaultsConfig, @@ -87,7 +86,7 @@ def validate_natively( # noqa: PLR0913 ) assert not isinstance(outcome, int) for error in filter(None, (outcome,)): - raise native_config_error(error=error, values=config.values) + raise error.exception(values=config.values) def build_config_natively( # noqa: PLR0913 @@ -127,7 +126,7 @@ def build_config_natively( # noqa: PLR0913 native_config_classes(session=session), ) for error in filter(_is_config_error, (built,)): - raise native_config_error(error=cast(_native.NativeConfigError, error)) + raise cast(_native.NativeConfigError, error).exception() return cast(CompileModelConfig, built) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index f67198973a..2c312977cd 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -12,10 +12,12 @@ build_manifest, build_native_declaration_contexts, build_scope_lookup, + collect_compile_diagnostics, discover_project_inputs, expand_sql_macros, infer_native_query_columns, project_lineage_views, + resolve_ref_references, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ExpressionInferenceProfile @@ -25,9 +27,6 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns -from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( - collect_compile_diagnostics, -) from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.models import ( CompileAdapterContext, @@ -63,7 +62,6 @@ ColumnTransformKind, InferredNullability, ) -from sqlbuild.compiler.planner._helpers.resolve.refs import resolve_ref_references from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py index 0ad9a64bbe..8e08025218 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py @@ -4,13 +4,13 @@ import pytest -from sqlbuild._native import watermark_type_is_compatible -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileModelConfig -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( +from sqlbuild._native import ( compute_cursor_bounds, resolve_effective_timestamp_grain, + watermark_type_is_compatible, ) +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import CompileModelConfig from sqlbuild.compiler.planner.models import CursorBounds, CursorInputRelation, ModelCursorSnapshot from sqlbuild.cursor_algebra.models import DateValue, TimestampValue from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py index 6b0e9aed81..6272cc9b6d 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild._native import assemble_project +from sqlbuild._native import assemble_project, scope_overlaps from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -20,7 +20,6 @@ CompileInputsView, TypedSqlValueRenderer, ) -from sqlbuild.compiler.planner._helpers.output.plan_entry import scope_overlaps from sqlbuild.sql_values.models import SqlLogicalType, SqlValue from sqlbuild.sql_values.types import SqlValueKind from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py deleted file mode 100644 index 91845ccf44..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py +++ /dev/null @@ -1,133 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.compiler.discovery.exceptions import DiscoveryConflictError -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile -from sqlbuild.compiler.path_defaults.main._select import select_path_default -from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( - FindMatchingPathDefaultErrorTestCase, - FindMatchingPathDefaultTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - ( - FindMatchingPathDefaultTestCase( - description="matches Windows-style model path against nested path default", - model_relative_path="models\\staging\\nested\\orders.sql", - path_defaults={"staging": {}, "staging/nested": {}}, - expected_path_default="staging/nested", - ), - FindMatchingPathDefaultTestCase( - description="recursive glob matches a staging directory prefix", - model_relative_path="models/commerce/partner_feed/staging/orders.sql", - path_defaults={"commerce/**/staging": {}}, - expected_path_default="commerce/**/staging", - ), - FindMatchingPathDefaultTestCase( - description="recursive glob matches zero path segments", - model_relative_path="models/commerce/staging/orders.sql", - path_defaults={"commerce/**/staging": {}}, - expected_path_default="commerce/**/staging", - ), - FindMatchingPathDefaultTestCase( - description="single glob consumes exactly one path segment", - model_relative_path="models/commerce/partner_feed/staging/orders.sql", - path_defaults={"commerce/*/staging": {}}, - expected_path_default="commerce/*/staging", - ), - FindMatchingPathDefaultTestCase( - description="single glob does not consume multiple path segments", - model_relative_path="models/commerce/eu/partner_feed/staging/orders.sql", - path_defaults={"commerce/*/staging": {}}, - expected_path_default=None, - ), - FindMatchingPathDefaultTestCase( - description="literal prefix outranks a matching recursive glob", - model_relative_path="models/commerce/partner_feed/staging/orders.sql", - path_defaults={ - "commerce/**/staging": {}, - "commerce/partner_feed/staging": {}, - }, - expected_path_default="commerce/partner_feed/staging", - ), - FindMatchingPathDefaultTestCase( - description="single segment glob outranks recursive glob", - model_relative_path="models/commerce/partner_feed/staging/orders.sql", - path_defaults={ - "commerce/**/staging": {}, - "commerce/*/staging": {}, - }, - expected_path_default="commerce/*/staging", - ), - FindMatchingPathDefaultTestCase( - description="recursive glob matches Windows-style model path", - model_relative_path="models\\commerce\\partner_feed\\staging\\orders.sql", - path_defaults={"commerce/**/staging": {}}, - expected_path_default="commerce/**/staging", - ), - ), - ids=lambda case: case.description, -) -def test_given_windows_style_model_path_when_finding_path_default_then_returns_nearest_match( - test_case: FindMatchingPathDefaultTestCase, -) -> None: - model_file: DiscoveredSqlModelFile = DiscoveredSqlModelFile( - file_path=Path("/repo") / test_case.model_relative_path, - relative_path=Path(test_case.model_relative_path), - contents="MODEL (description 'Test model.');\n\nselect 1\n", - header_values={}, - header_column_locations={}, - output_column_locations={}, - query_sql="select 1", - ) - - path_default: str | None = select_path_default( - model_path=str(model_file.relative_path), path_keys=tuple(test_case.path_defaults) - ).selected_key - - assert path_default == test_case.expected_path_default - - -@pytest.mark.parametrize( - "test_case", - ( - FindMatchingPathDefaultErrorTestCase( - description="equally specific matching globs are ambiguous regardless of values", - model_relative_path="models/commerce/eu/staging/orders.sql", - path_defaults={ - "commerce/*/staging": {"schema": "shared"}, - "commerce/eu/*": {"schema": "shared"}, - }, - expected_error_fragment=( - r"matches equally specific path_defaults keys: " - r"'commerce/\*/staging', 'commerce/eu/\*'" - ), - ), - ), - ids=lambda case: case.description, -) -def test_given_ambiguous_globs_when_finding_path_default_then_raises_structured_error( - test_case: FindMatchingPathDefaultErrorTestCase, -) -> None: - model_file: DiscoveredSqlModelFile = DiscoveredSqlModelFile( - file_path=Path("/repo") / test_case.model_relative_path, - relative_path=Path(test_case.model_relative_path), - contents="MODEL (description 'Test model.');\n\nselect 1\n", - header_values={}, - header_column_locations={}, - output_column_locations={}, - query_sql="select 1", - ) - - with pytest.raises(DiscoveryConflictError, match=test_case.expected_error_fragment) as error: - select_path_default( - model_path=str(model_file.relative_path), path_keys=tuple(test_case.path_defaults) - ) - - assert error.value.code == "D007" - assert error.value.help is not None diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py index 64bb9abacf..fcb2625574 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py @@ -6,13 +6,12 @@ import pytest -from sqlbuild._native import extract_sql_references +from sqlbuild._native import collect_compile_diagnostics, extract_sql_references from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile._helpers.diagnostics.collector import collect_compile_diagnostics from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileSqlReference from sqlbuild.compiler.references.types import SqlReferenceKind diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py index b418fe8604..8830420024 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py @@ -3,10 +3,10 @@ import pytest from sqlbuild._native import ( + UnicodeEnvironment, expand_effective_vars, expand_template_data, ) -from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpandTemplateDataErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py index f6b19d46e7..19f4aba554 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py @@ -1,7 +1,6 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.yml.yaml_classes import yaml_entry_classes from sqlbuild.compiler.discovery.constants import ( REMOVED_SQL_MODEL_HEADER_KEYS, SQL_MODEL_HEADER_KEYS, @@ -95,7 +94,6 @@ def parse_sources_yml(*, contents: str, file_path: Path) -> tuple[SourceEntry, . return _native.parse_sources_document( _native.load_yaml_document(str(file_path), contents, "source"), str(file_path), - yaml_entry_classes(), ) @@ -107,5 +105,4 @@ def parse_schema_yml( return (), _native.parse_schema_document( _native.load_yaml_document(str(file_path), contents, "schema"), str(file_path), - yaml_entry_classes(), ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/main/test_yaml_entry_line.py b/tests/unit/src/sqlbuild/compiler/discovery/main/test_yaml_entry_line.py index 278ad2b932..c1c98d2e03 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/main/test_yaml_entry_line.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/main/test_yaml_entry_line.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.discovery.main._yaml_entry_line import yaml_entry_line +from sqlbuild._native import yaml_entry_line from tests.unit.src.sqlbuild.compiler.discovery.main._test_types import YamlEntryLineTestCase _SOURCES: str = ( diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index deee9d90ef..87815a8441 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -9,16 +9,15 @@ import pytest import sqlbuild._native as _native +from sqlbuild._native import BuildPhaseTimingTracker from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline.main import _plan_result as plan_result +from sqlbuild.compiler.pipeline._helpers import materializations from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker from sqlbuild.diagnostics.models import PartialBuildPhaseTimings from sqlbuild.rule_engine._helpers.run import rule_threads from sqlbuild.rule_engine.models import RulesRunResult -from sqlbuild.runtime.contracts.main import open_connection from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import PipelinePhaseTimingTestCase @@ -28,14 +27,14 @@ [ PipelinePhaseTimingTestCase( description="slow planning connection is excluded from direct compilation", - clock_values=(0.0, 0.0, 2.0, 2.0, 3.0, 13.0, 13.0), + clock_values=(0.0, 0.0, 2.0, 2.0, 3.0, 3.0, 13.0, 13.0), expected_compile_seconds=2.0, expected_planning_seconds=10.0, expected_total_seconds=13.0, ), PipelinePhaseTimingTestCase( description="rules wall time is reported and excluded from compile and planning", - clock_values=(0.0, 0.0, 2.0, 2.0, 3.0, 3.0, 13.0, 13.0), + clock_values=(0.0, 0.0, 2.0, 2.0, 3.0, 3.0, 3.0, 13.0, 13.0), expected_compile_seconds=2.0, expected_planning_seconds=10.0, expected_total_seconds=13.0, @@ -66,8 +65,19 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di adapter: Mock = Mock() monkeypatch.setattr(time, "monotonic", monotonic) monkeypatch.setattr(_native, "build_compiled_project", Mock(return_value=project)) - monkeypatch.setattr(open_connection, "open_connection_with_hooks", Mock(return_value=object())) - monkeypatch.setattr(plan_result, "build_pipeline_result", Mock(return_value=pipeline_result)) + monkeypatch.setattr(_native, "build_execution_plan", Mock(return_value=PlanOutput())) + monkeypatch.setattr(materializations, "load_custom_materializations", Mock(return_value={})) + monkeypatch.setattr( + _native, + "build_python_run_plan_outputs", + Mock( + return_value=Mock( + plan_output=pipeline_result.plan_output, + selected_python_node_names=frozenset(), + python_plan_entries=(), + ) + ), + ) (tmp_path / "sqlbuild_project.toml").write_text('name = "timing"\n', encoding="utf-8") monkeypatch.setattr(_native, "resolve_configured_rule_selection", Mock(return_value=None)) monkeypatch.setattr( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_old_name_rows.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_old_name_rows.py index 443abdb3a2..5029617c24 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_old_name_rows.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_old_name_rows.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import old_name_rows -from sqlbuild.presentation.classes.cli_style import CliStyle +from sqlbuild._native import CliStyle, old_name_rows from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( OldNameRowsTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py index 3a41b522c7..2aac608caf 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py @@ -7,13 +7,10 @@ import pytest +from sqlbuild._native import apply_typed_cursor_replay_policy, compute_cursor_bounds from sqlbuild._native import cursor_observed_partition as observed_partition from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - apply_typed_cursor_replay_policy, - compute_cursor_bounds, -) from sqlbuild.compiler.planner.constants import ( MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py index 1ac4c873f5..4a6a6ee82c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py @@ -2,9 +2,9 @@ import pytest +from sqlbuild._native import resolve_model_sql from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_model_sql from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( BackfillResult, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_override_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_override_types.py index 3bef326593..19522990df 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_override_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_override_types.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.planner._helpers.resolve.cursor import type_cursor_overrides +from sqlbuild._native import type_cursor_overrides from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import CursorOverrides from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py index c71184bd0b..14311bc6e5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py @@ -6,10 +6,10 @@ import pytest +from sqlbuild._native import resolve_model_hook_entries from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry -from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_model_hook_entries from sqlbuild.compiler.planner.models import ModelPlanContext from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py index 95886abb32..e790e6ecb7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py @@ -4,7 +4,15 @@ import pytest -from sqlbuild._native import DbtCompileReferenceResolver, build_dbt_manifest_index +from sqlbuild._native import ( + DbtCompileReferenceResolver, + apply_deferred_locations, + build_dbt_manifest_index, + resolve_dbt_ref_references, + resolve_ref_references, + resolve_table_function_references, + resolve_udf_references, +) from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -12,13 +20,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner._helpers.resolve.refs import ( - apply_deferred_locations, - resolve_dbt_ref_references, - resolve_ref_references, - resolve_table_function_references, - resolve_udf_references, -) from sqlbuild.compiler.planner.models import CursorBounds from sqlbuild.integrations.dbt.models import DbtManifestIndex from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py index f77d96815e..62a2c40c5c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py @@ -4,6 +4,7 @@ import pytest +from sqlbuild._native import resolve_source_references from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -11,9 +12,6 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.planner._helpers.resolve.sources import ( - resolve_source_references, -) from sqlbuild.compiler.planner.models import CursorBounds from sqlbuild.spec.contracts.models import SourceColumnEntry, SourceEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index 5761e52fc5..b6c625c5b3 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -6,8 +6,11 @@ import pytest from sqlbuild._native import ( + build_test_command_plan, plan_compiled_sql_test_artifacts, project_with, + resolve_static_command_scope, + resolve_static_relation_context, sql_test_plan_error_messages, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -27,9 +30,6 @@ from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.commands._relations import resolve_static_relation_context -from sqlbuild.compiler.planner.main.commands._scope import resolve_static_command_scope -from sqlbuild.compiler.planner.main.commands._sql_test import build_test_command_plan from sqlbuild.compiler.planner.models import ( ChainStep, NativeSqlTestPlan, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py index 41cd712f21..7ad94b1446 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import resolve_cursor_overrides from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner._helpers.output.plan_entry import resolve_cursor_overrides from sqlbuild.compiler.planner.models import CursorOverrides from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorOverrideResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py index 55d33f98f9..351f03d67d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import compute_cursor_bounds from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.compiler.planner._helpers.resolve.cursor import compute_cursor_bounds from sqlbuild.compiler.planner.models import CursorBounds, ModelCursorSnapshot from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorStartBoundsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index 76098d4a9a..832fda88be 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -8,6 +8,7 @@ import sqlbuild._native as sql_test_assembly from sqlbuild import _native as fixture_completion from sqlbuild._native import ( + build_selected_test_entries, compiled_model_record, project_with, ) @@ -21,8 +22,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner._helpers.output import plan_output -from sqlbuild.compiler.planner._helpers.output.plan_output import build_selected_test_entries from sqlbuild.compiler.planner.classes import fixture_column_inferences as fixture_inferences from sqlbuild.compiler.planner.models import ( FixtureColumnMetadata, @@ -101,7 +100,7 @@ def test_given_many_tests_when_building_entries_then_project_fixture_metadata_is settings=SettingsConfig(sql_analysis=True), ) metadata_builder: Mock = Mock(wraps=fixture_completion.build_relation_fixture_context) - monkeypatch.setattr(plan_output, "build_relation_fixture_context", metadata_builder) + monkeypatch.setattr(fixture_completion, "build_relation_fixture_context", metadata_builder) fixture_completion.fixture_read_cache_info(reset=True) topology_builder: Mock = Mock(wraps=sql_test_assembly.resolve_sql_test_model_chains) monkeypatch.setattr(sql_test_assembly, "resolve_sql_test_model_chains", topology_builder) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py index d163308c0a..1616717d3f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild._native import compute_microbatch_range, compute_plan_cursor_bounds from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -15,11 +16,6 @@ CompileModelConfig, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.planner._helpers.output.plan_entry import ( - _compute_microbatch_range, - _compute_plan_cursor_bounds, - _MicrobatchRangeInputs, -) from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL from sqlbuild.compiler.planner.models import ( BackfillResult, @@ -86,7 +82,7 @@ def test_given_planner_owned_microbatch_with_cursor_end_when_planning_then_senti ) backfill: BackfillResult = BackfillResult(action=BackfillAction.FORWARD_ONLY) - plan_bounds: CursorBounds | None = _compute_plan_cursor_bounds( + plan_bounds: CursorBounds | None = compute_plan_cursor_bounds( model=model, snapshot=snapshot, backfill=backfill, @@ -95,22 +91,20 @@ def test_given_planner_owned_microbatch_with_cursor_end_when_planning_then_senti end_cursor_override=None, runtime_owned_cursor_bounds=False, ) - microbatch_range: CursorBounds | None = _compute_microbatch_range( + microbatch_range: CursorBounds | None = compute_microbatch_range( model=model, - inputs=_MicrobatchRangeInputs( - snapshot=snapshot, - backfill=backfill, - start_cursor_override=None, - end_cursor_override=None, - runtime_owned_cursor_bounds=False, - cursor_input_relations=( - CursorInputRelation(relation="main.raw_orders", cursor_column="cursor_value"), - ), - future_cursor_config=None, - start_cursor_config=None, - invocation_time=None, - full_refresh=False, + snapshot=snapshot, + backfill=backfill, + start_cursor_override=None, + end_cursor_override=None, + runtime_owned_cursor_bounds=False, + cursor_input_relations=( + CursorInputRelation(relation="main.raw_orders", cursor_column="cursor_value"), ), + future_cursor_config=None, + start_cursor_config=None, + invocation_time=None, + full_refresh=False, ) assert plan_bounds == CursorBounds( @@ -161,7 +155,7 @@ def test_given_explicit_cursor_overrides_without_snapshot_when_planning_then_use ), ) - bounds: CursorBounds | None = _compute_plan_cursor_bounds( + bounds: CursorBounds | None = compute_plan_cursor_bounds( model=model, snapshot=WarehouseSnapshot(), backfill=BackfillResult(action=BackfillAction.BOUNDED, duration="7d"), @@ -257,7 +251,7 @@ def test_given_runtime_owned_missing_snapshot_when_overrides_complete_then_all_r } ), ) - plan_bounds: CursorBounds | None = _compute_plan_cursor_bounds( + plan_bounds: CursorBounds | None = compute_plan_cursor_bounds( model=model, snapshot=snapshot, backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), @@ -266,20 +260,18 @@ def test_given_runtime_owned_missing_snapshot_when_overrides_complete_then_all_r end_cursor_override=test_case.end_override, runtime_owned_cursor_bounds=True, ) - microbatch_range: CursorBounds | None = _compute_microbatch_range( + microbatch_range: CursorBounds | None = compute_microbatch_range( model=microbatch_model, - inputs=_MicrobatchRangeInputs( - snapshot=snapshot, - backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), - start_cursor_override=test_case.start_override, - end_cursor_override=test_case.end_override, - runtime_owned_cursor_bounds=True, - cursor_input_relations=(), - future_cursor_config=None, - start_cursor_config=None, - invocation_time=None, - full_refresh=False, - ), + snapshot=snapshot, + backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + start_cursor_override=test_case.start_override, + end_cursor_override=test_case.end_override, + runtime_owned_cursor_bounds=True, + cursor_input_relations=(), + future_cursor_config=None, + start_cursor_config=None, + invocation_time=None, + full_refresh=False, ) assert plan_bounds == test_case.expected_bounds @@ -355,28 +347,26 @@ def test_given_planner_owned_model_backed_input_when_grain_is_coarser_then_range } ) - microbatch_range: CursorBounds | None = _compute_microbatch_range( + microbatch_range: CursorBounds | None = compute_microbatch_range( model=model, - inputs=_MicrobatchRangeInputs( - snapshot=snapshot, - backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), - start_cursor_override=None, - end_cursor_override=None, - runtime_owned_cursor_bounds=False, - cursor_input_relations=( - CursorInputRelation( - relation="main.daily_rollup", - cursor_column="activity_day", - cursor_grain=test_case.upstream_grain, - is_model_backed=True, - is_runtime_produced=False, - ), + snapshot=snapshot, + backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + start_cursor_override=None, + end_cursor_override=None, + runtime_owned_cursor_bounds=False, + cursor_input_relations=( + CursorInputRelation( + relation="main.daily_rollup", + cursor_column="activity_day", + cursor_grain=test_case.upstream_grain, + is_model_backed=True, + is_runtime_produced=False, ), - future_cursor_config=None, - start_cursor_config=None, - invocation_time=None, - full_refresh=False, ), + future_cursor_config=None, + start_cursor_config=None, + invocation_time=None, + full_refresh=False, ) assert microbatch_range == test_case.expected_bounds @@ -421,7 +411,7 @@ def test_given_snapshot_cursor_grain_when_planning_then_dml_bound_matches_query_ ), ) - bounds: CursorBounds | None = _compute_plan_cursor_bounds( + bounds: CursorBounds | None = compute_plan_cursor_bounds( model=model, snapshot=WarehouseSnapshot( cursor_snapshots={ diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py index f158cefe5b..e106f642a7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py @@ -7,6 +7,7 @@ build_scenario_hash_index, build_scenario_relation_map, compute_scenario_hash_prefix, + is_scenario_artifact_physical_name, parse_scenario_artifact_physical_name, ) from sqlbuild.compiler.planner.constants import ( @@ -14,9 +15,6 @@ SCENARIO_PLAN_RELATION_COLLISION, ) from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.scenarios.is_scenario_artifact_physical_name import ( - is_scenario_artifact_physical_name, -) from sqlbuild.compiler.planner.models import ( ParsedScenarioArtifactName, ScenarioArtifactIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py index ca6736d1da..486e3bba2e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( build_cursor_input_relations, + build_planner_relations_context, gather_source_columns, runtime_cursor_producer_names, validate_source_cursor_input_columns, @@ -25,7 +26,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( CursorInputRelation, diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_background_sql_test_planning.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py similarity index 83% rename from tests/unit/src/sqlbuild/compiler/planner/classes/test_background_sql_test_planning.py rename to tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py index 83600a755c..0b8f24f86e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_background_sql_test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py @@ -1,15 +1,14 @@ from __future__ import annotations import threading +from concurrent.futures import Future import pytest -import sqlbuild.compiler.planner.classes.background_sql_test_planning as background_module +import sqlbuild._native as native_module +from sqlbuild._native import start_sql_test_planning from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.classes.background_sql_test_planning import ( - BackgroundSqlTestPlanning, -) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import PlannedSqlTests, PlanWarning, SqlTestSelection from sqlbuild.compiler.planner.types import WarningSeverity @@ -55,16 +54,16 @@ def plan( calls.append(selected_keys) return _PLANNED - monkeypatch.setattr(background_module, "plan_selected_sql_tests", plan) + monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) - with BackgroundSqlTestPlanning( + planning: Future[PlannedSqlTests] = start_sql_test_planning( sql_test_selection=SqlTestSelection(), project=object(), # ty: ignore[invalid-argument-type] adapter=object(), # ty: ignore[invalid-argument-type] selected_keys=_KEYS, enabled=test_case.enabled, - ) as planning: - result: PlannedSqlTests = planning.result() + ) + result: PlannedSqlTests = planning.result() assert result == _PLANNED assert calls == [_KEYS] * test_case.expected_planner_calls @@ -97,16 +96,16 @@ def plan( calls.append(selected_keys) return _PLANNED - monkeypatch.setattr(background_module, "plan_selected_sql_tests", plan) + monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) - with BackgroundSqlTestPlanning( + planning: Future[PlannedSqlTests] = start_sql_test_planning( sql_test_selection=SqlTestSelection(), project=object(), # ty: ignore[invalid-argument-type] adapter=object(), # ty: ignore[invalid-argument-type] selected_keys=_KEYS, enabled=test_case.enabled, - ) as planning: - result: PlannedSqlTests = planning.result() + ) + result: PlannedSqlTests = planning.result() assert result == PlannedSqlTests(selected_keys=_KEYS) assert len(calls) == test_case.expected_planner_calls @@ -137,17 +136,17 @@ def plan( del project, adapter, selected_keys, sql_test_selection raise PlannerInputError(str(test_case.expected_error)) - monkeypatch.setattr(background_module, "plan_selected_sql_tests", plan) + monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) - with BackgroundSqlTestPlanning( + planning: Future[PlannedSqlTests] = start_sql_test_planning( sql_test_selection=SqlTestSelection(), project=object(), # ty: ignore[invalid-argument-type] adapter=object(), # ty: ignore[invalid-argument-type] selected_keys=_KEYS, enabled=test_case.enabled, - ) as planning: - with pytest.raises(PlannerInputError, match=str(test_case.expected_error)): - planning.result() + ) + with pytest.raises(PlannerInputError, match=str(test_case.expected_error)): + planning.result() @pytest.mark.parametrize( @@ -179,20 +178,21 @@ def plan( release.wait(timeout=5) return PlannedSqlTests(selected_keys=selected_keys) - monkeypatch.setattr(background_module, "plan_selected_sql_tests", plan) + monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) - with BackgroundSqlTestPlanning( + planning: Future[PlannedSqlTests] = start_sql_test_planning( sql_test_selection=SqlTestSelection(), project=object(), # ty: ignore[invalid-argument-type] adapter=object(), # ty: ignore[invalid-argument-type] selected_keys=_KEYS, enabled=test_case.enabled, - ): - assert started.wait(timeout=5) - daemon_by_name: dict[str, bool] = { - thread.name: thread.daemon for thread in threading.enumerate() - } + ) + assert started.wait(timeout=5) + daemon_by_name: dict[str, bool] = { + thread.name: thread.daemon for thread in threading.enumerate() + } release.set() + planning.result() assert daemon_by_name["sqlbuild-test-planning"] is True assert test_case.expected_planner_calls == 1 diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py index e3ddb6f0ba..44a05fa22b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild._native import format_plan +from sqlbuild._native import build_display_only_sqlbuild_plan, format_plan from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -15,9 +15,6 @@ CompileModelConfig, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.planner.main.execution.display_plan import ( - build_display_only_sqlbuild_plan, -) from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput from sqlbuild.compiler.planner.types import PlanAction, PlanReason from tests.unit.src.sqlbuild.compiler.planner.main.execution._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index 6fd2cae9c2..b4dd031f7c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -6,6 +6,7 @@ from typing import Any from sqlbuild._native import ( + build_execution_plan, build_version_identity_metadata_json, compute_query_hash, write_fingerprint, @@ -24,7 +25,6 @@ CompiledResourceType, ) from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.main.execution.execution import build_execution_plan from sqlbuild.compiler.planner.models import ( DeferralInputs, PlannerOverrides, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py index 7091b8cc87..33ec9ff9f4 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import format_pre_build_work from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.migrations.types import ( @@ -11,7 +12,6 @@ MigrationDiscovery, MigrationPromotion, ) -from sqlbuild.compiler.planner.main.pre_build.pre_build_work_display import format_pre_build_work from sqlbuild.compiler.planner.models import ( ColumnMigrationPlanEntry, ModelMigrationPlanEntry, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py index ea481f74eb..04e9cc6423 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py @@ -4,14 +4,13 @@ import pytest -from sqlbuild._native import format_plan +from sqlbuild._native import format_plan, format_pre_build_work from sqlbuild.compiler.migrations.types import ( ColumnMigrationDecision, MigrationCompatibility, MigrationDecision, MigrationDiscovery, ) -from sqlbuild.compiler.planner.main.pre_build.pre_build_work_display import format_pre_build_work from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.presentation.models import DisplayOptions from tests.unit.src.sqlbuild.compiler.planner.main.pre_build._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_defer_clone_traversal.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_defer_clone_traversal.py index fe6beb6023..506301703b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_defer_clone_traversal.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_defer_clone_traversal.py @@ -1,11 +1,6 @@ import pytest -from sqlbuild.compiler.planner.main.clone.resolve_clone_boundary import ( - resolve_clone_boundary, -) -from sqlbuild.compiler.planner.main.clone.resolve_skipped_view_chain import ( - resolve_skipped_view_chain, -) +from sqlbuild._native import resolve_clone_boundary, resolve_skipped_view_chain from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( CloneBoundaryTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py index 12f9cc2855..62d897ec76 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py @@ -5,9 +5,9 @@ import pytest from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token +from sqlbuild._native import future_cursor_cap_warning +from sqlbuild._native import resolve_future_cursor_safety as apply_future_cursor_safety from sqlbuild.compiler.planner.exceptions import FutureCursorSafetyError -from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety -from sqlbuild.compiler.planner.main.execution.future_cursor_warning import future_cursor_cap_warning from sqlbuild.compiler.planner.models import CursorBounds from sqlbuild.compiler.planner.types import CursorGrain, CursorType from sqlbuild.spec.contracts.models import FutureCursorsConfig diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py index 34b2183d86..6c10df7d30 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py @@ -5,10 +5,10 @@ import pytest +from sqlbuild._native import compute_cursor_bounds from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.compiler.planner._helpers.resolve.cursor import compute_cursor_bounds +from sqlbuild._native import resolve_future_cursor_safety as apply_future_cursor_safety from sqlbuild.compiler.planner.exceptions import MaximumAutomaticStartError -from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety from sqlbuild.compiler.planner.models import ( CursorBounds, MaximumStartPolicyInputs, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py index 1107511c5a..0398232fca 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py @@ -10,19 +10,19 @@ import pytest from sqlbuild._native import ( + build_audit_command_plan, + build_seed_command_plan, + build_test_command_plan, compiled_model_record, project_with, + read_selected_seed_fingerprints, + resolve_static_command_scope, + resolve_static_relation_context, ) from sqlbuild.compiler.compile.models import CompileModelConfig from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledSeedView from sqlbuild.compiler.discovery.models import DiscoveredSeedFile from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.main.commands._audit import build_audit_command_plan -from sqlbuild.compiler.planner.main.commands._relations import resolve_static_relation_context -from sqlbuild.compiler.planner.main.commands._scope import resolve_static_command_scope -from sqlbuild.compiler.planner.main.commands._seed import build_seed_command_plan -from sqlbuild.compiler.planner.main.commands._seed_state import read_selected_seed_fingerprints -from sqlbuild.compiler.planner.main.commands._sql_test import build_test_command_plan from sqlbuild.compiler.planner.models import ( PlannerRelationsContext, PlannerScope, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py index 54e38e925e..64ec4f7b5f 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py @@ -9,8 +9,11 @@ import pytest -from sqlbuild._native import load_or_build_scope_index, scope_index_fingerprint -from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory +from sqlbuild._native import ( + compiler_cache_directory, + load_or_build_scope_index, + scope_index_fingerprint, +) from sqlbuild.compiler.scopes.constants import SCOPE_CACHE_DIRECTORY_NAME, SCOPE_CACHE_FILENAME from sqlbuild.compiler.scopes.models import DeclarationIdentity, ScopeIndex from sqlbuild.compiler.scopes.types import ( diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py index 1eb327b456..11edc5e466 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py @@ -5,13 +5,13 @@ import pytest -from sqlbuild._native import normalize_source_freshness_data_version -from sqlbuild.adapter.contract.models import TableFreshnessMetadata -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessObservationError -from sqlbuild.compiler.source_freshness.main.observation import ( +from sqlbuild._native import ( + SourceFreshnessObservationError, + normalize_source_freshness_data_version, observe_configured_source_freshness, ) +from sqlbuild.adapter.contract.models import TableFreshnessMetadata +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.source_freshness.models import SourceFreshnessObservation from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py index 8ca26d561c..f0ec34eb25 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py @@ -5,7 +5,11 @@ from types import MappingProxyType from typing import Any -from sqlbuild._native import source_freshness_data_version_hash +from sqlbuild._native import ( + build_source_freshness_read_latest_sql, + source_freshness_data_version_hash, + write_source_freshness_records, +) from sqlbuild.adapter.contract.models import ( RelationLookup, TableFreshnessMetadata, @@ -16,9 +20,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.source_freshness._helpers.sql import build_read_latest_sql from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME -from sqlbuild.compiler.source_freshness.main.write import write_source_freshness_records from sqlbuild.compiler.source_freshness.models import ( SourceFreshnessIdentity, SourceFreshnessRecord, @@ -100,7 +102,7 @@ def render_qualified_name(*, database: str | None, schema: str | None, name: str def render_read_latest_sql(*, database: str | None, schema: str) -> str: - return build_read_latest_sql( + return build_source_freshness_read_latest_sql( database=database, schema=schema, render_qualified_name=render_qualified_name, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py index 08a5bd4908..c49624b344 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py @@ -4,13 +4,12 @@ import pytest +from sqlbuild._native import ( + build_source_freshness_create_table_sql, + build_source_freshness_read_latest_sql, +) from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness._helpers.sql import ( - build_create_table_sql, - build_qualified_table_name, - build_read_latest_sql, -) from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from tests.unit.src.sqlbuild.compiler.source_freshness.main.shared._helpers._test_types import ( BuildSourceFreshnessSqlTestCase, @@ -35,7 +34,7 @@ def test_given_schema_when_building_qualified_name_then_returns_expected( test_case: BuildSourceFreshnessSqlTestCase, ) -> None: - result: str = build_qualified_table_name( + result: str = build_source_freshness_read_latest_sql( database=test_case.database, schema=test_case.schema, render_qualified_name=RENDER_QUALIFIED_NAME, @@ -74,7 +73,7 @@ def test_given_schema_when_building_qualified_name_then_returns_expected( def test_given_schema_when_building_create_table_sql_then_contains_expected_fragments( test_case: BuildSourceFreshnessSqlTestCase, ) -> None: - result: str = build_create_table_sql( + result: str = build_source_freshness_create_table_sql( database=test_case.database, schema=test_case.schema, render_qualified_name=RENDER_QUALIFIED_NAME, @@ -107,7 +106,7 @@ def test_given_schema_when_building_create_table_sql_then_contains_expected_frag def test_given_schema_when_building_read_latest_sql_then_contains_windowed_latest_query( test_case: BuildSourceFreshnessSqlTestCase, ) -> None: - result: str = build_read_latest_sql( + result: str = build_source_freshness_read_latest_sql( database=test_case.database, schema=test_case.schema, render_qualified_name=RENDER_QUALIFIED_NAME, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py index 7d087e4ad2..8c7a31b198 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py @@ -5,13 +5,11 @@ import pytest -from sqlbuild.adapter.contract.models import TableFreshnessMetadata -from sqlbuild.compiler.source_freshness.main._planning import ( +from sqlbuild._native import ( build_direct_source_freshness_planning_result, -) -from sqlbuild.compiler.source_freshness.main.adapter_observation import ( observe_adapter_sources_freshness, ) +from sqlbuild.adapter.contract.models import TableFreshnessMetadata from sqlbuild.compiler.source_freshness.models import ( AdapterSourceFreshnessBatch, DirectSourceFreshnessPlanningResult, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py index 1532b9780b..b81dbdfdb8 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py @@ -5,11 +5,11 @@ import pytest -from sqlbuild.adapter.contract.models import QueryResult, TableFreshnessMetadata -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.main.observation import ( +from sqlbuild._native import ( observe_configured_source_freshness, ) +from sqlbuild.adapter.contract.models import QueryResult, TableFreshnessMetadata +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.source_freshness.models import SourceFreshnessObservation from sqlbuild.observability import ( EventDispatcher, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py index 6d56c58452..06ec5f6358 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py @@ -6,14 +6,14 @@ import pytest -from sqlbuild._native import source_freshness_data_version_hash +from sqlbuild._native import ( + build_direct_source_freshness_planning_result, + source_freshness_data_version_hash, + write_source_freshness_records, +) from sqlbuild.adapter.contract.models import TableFreshnessMetadata, TableFreshnessRequest from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.main._planning import ( - build_direct_source_freshness_planning_result, -) -from sqlbuild.compiler.source_freshness.main.write import write_source_freshness_records from sqlbuild.compiler.source_freshness.models import ( DirectSourceFreshnessPlanningResult, SourceFreshnessRecord, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_propagation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_propagation.py index 9b04c178f6..547a5375f9 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_propagation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_propagation.py @@ -2,10 +2,10 @@ import pytest -from sqlbuild.compiler.planner.models import PlannerScope -from sqlbuild.compiler.source_freshness.main._propagation import ( +from sqlbuild._native import ( build_direct_source_freshness_propagation_result, ) +from sqlbuild.compiler.planner.models import PlannerScope from sqlbuild.compiler.source_freshness.models import ( DirectSourceFreshnessPlanningResult, DirectSourceFreshnessPropagationResult, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py index f94c5664be..f889c13e35 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py @@ -4,10 +4,12 @@ import pytest +from sqlbuild._native import ( + SourceFreshnessInputError, + read_latest_source_freshness, + write_source_freshness_records, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessInputError -from sqlbuild.compiler.source_freshness.main.read import read_latest_source_freshness -from sqlbuild.compiler.source_freshness.main.write import write_source_freshness_records from sqlbuild.compiler.source_freshness.models import ( SourceFreshnessIdentity, SourceFreshnessRecord, diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py index 1e58ac30a7..040d0a2090 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py @@ -75,3 +75,22 @@ def normalize_sql_for_polyglot(*, sql: str, dialect: str | None) -> str: return cast(NativePositionsModule, _native).normalize_dialect_sql( sql=sql, dialect=dialect or "generic" ) + + +def normalize_analysis_sql( + *, + sql: str, + dialect: str | None, + stubs: Mapping[str, str] | None = None, + placeholders: Mapping[str, str] | None = None, +) -> str: + """Normalize one analysis SQL natively.""" + + return cast(NativePositionsModule, _native).normalize_analysis_sql( + { + "sql": sql, + "dialect": dialect or "generic", + "stubs": dict(stubs or {}), + "placeholders": dict(placeholders or {}), + } + ) diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis.py index 038b660a85..0975cc0988 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis.py @@ -2,10 +2,10 @@ import pytest -from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql from tests.unit.src.sqlbuild.compiler.sql_analysis.main._test_types import ( PolyglotSqlNormalizationTestCase, ) +from tests.unit.src.sqlbuild.compiler.sql_analysis.main.helpers import normalize_analysis_sql @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis_batch.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis_batch.py index 74f0ebef88..91468e9505 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis_batch.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis_batch.py @@ -2,13 +2,13 @@ import pytest -from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql from tests.unit.src.sqlbuild.compiler.sql_analysis.main._test_types import ( NormalizationBatchFailureTestCase, NormalizationBatchTestCase, NormalizationChunkTestCase, ) from tests.unit.src.sqlbuild.compiler.sql_analysis.main.helpers import ( + normalize_analysis_sql, normalize_analysis_sql_results, pool_catalog, without_catalog, diff --git a/tests/unit/src/sqlbuild/cost/test_allocation.py b/tests/unit/src/sqlbuild/cost/test_allocation.py index 9193b89449..b6e2c87032 100644 --- a/tests/unit/src/sqlbuild/cost/test_allocation.py +++ b/tests/unit/src/sqlbuild/cost/test_allocation.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.cost._helpers.allocation import allocate_run_cost +from sqlbuild._native import allocate_run_cost from sqlbuild.cost.models import QueryCostObservation, RunCostSummary from sqlbuild.cost.types import CostStatus from tests.unit.src.sqlbuild.cost._test_types import AllocationTestCase diff --git a/tests/unit/src/sqlbuild/cost/test_context.py b/tests/unit/src/sqlbuild/cost/test_context.py index daff103809..28ad5accbb 100644 --- a/tests/unit/src/sqlbuild/cost/test_context.py +++ b/tests/unit/src/sqlbuild/cost/test_context.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.cost.classes.cost_context import CostContext +from sqlbuild._native import CostContext from sqlbuild.cost.models import CostResourceContext from tests.unit.src.sqlbuild.cost._test_types import CostResourceScopeTestCase diff --git a/tests/unit/src/sqlbuild/cost/test_snowflake.py b/tests/unit/src/sqlbuild/cost/test_snowflake.py index 173fdadbc6..0a558b5c77 100644 --- a/tests/unit/src/sqlbuild/cost/test_snowflake.py +++ b/tests/unit/src/sqlbuild/cost/test_snowflake.py @@ -9,15 +9,14 @@ import pytest -from sqlbuild.cost._helpers.snowflake import ( - _is_metadata_only_statement, - _parse_sqlbuild_tag, - _query_observations, - _render_open_query_history_sql, - _render_query_history_sql, +from sqlbuild._native import ( + COST_TELEMETRY_HEALTH, + collect_snowflake_cost, + parse_sqlbuild_query_tag, + render_snowflake_query_history_sql, + snowflake_query_observations, + snowflake_statement_is_metadata_only, ) -from sqlbuild.cost.constants import COST_TELEMETRY_HEALTH -from sqlbuild.cost.main.collection import collect_snowflake_cost from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostStatus from tests.unit.src.sqlbuild.cost._test_types import ( @@ -88,7 +87,7 @@ def execute(self, sql: str, **kwargs: Any) -> _Cursor: def test_given_time_range_when_rendering_query_history_then_uses_valid_bounded_table_function( test_case: RenderQueryHistorySqlTestCase, ) -> None: - sql: str = _render_query_history_sql( + sql: str = render_snowflake_query_history_sql( database="analytics", started_at=datetime(2026, 8, 23, 10, 0, tzinfo=UTC), completed_at=datetime(2026, 8, 23, 10, 5, tzinfo=UTC), @@ -123,7 +122,7 @@ def test_given_time_range_when_rendering_query_history_then_uses_valid_bounded_t def test_given_start_time_when_rendering_open_history_then_running_queries_are_requested( test_case: RenderQueryHistorySqlTestCase, ) -> None: - sql: str = _render_open_query_history_sql( + sql: str = render_snowflake_query_history_sql( database="analytics", started_at=datetime(2026, 8, 23, 10, 0, tzinfo=UTC), classification_query_ids=frozenset({"q-create"}), @@ -148,7 +147,7 @@ def test_given_start_time_when_rendering_open_history_then_running_queries_are_r def test_given_no_classifiable_create_ids_when_rendering_history_then_query_text_is_masked( test_case: RenderQueryHistorySqlTestCase, ) -> None: - sql: str = _render_open_query_history_sql( + sql: str = render_snowflake_query_history_sql( started_at=datetime(2026, 8, 23, 10, 0, tzinfo=UTC), classification_query_ids=frozenset(), ) @@ -224,7 +223,6 @@ def test_given_exact_create_hook_when_collecting_then_hook_query_text_is_not_req ) def test_given_saturated_open_history_when_bounded_fallback_is_complete_then_not_saturated( test_case: SnowflakeSaturationTestCase, - monkeypatch: pytest.MonkeyPatch, ) -> None: origin: datetime = datetime(2026, 8, 23, 10, tzinfo=UTC) row: tuple[Any, ...] = ( @@ -247,9 +245,9 @@ def test_given_saturated_open_history_when_bounded_fallback_is_complete_then_not connection: _SequencedConnection = _SequencedConnection( ((row, ("q-open-2", *row[1:])), (bounded_row,)) ) - monkeypatch.setattr("sqlbuild.cost._helpers.snowflake._RESULT_LIMIT", 2) - _, observed_query_ids, _, saturated = _query_observations( + _, observed_query_ids, _, saturated = snowflake_query_observations( + result_limit=2, connection=connection, started_at=origin, completed_at=origin + timedelta(seconds=1), @@ -277,8 +275,6 @@ def test_given_saturated_open_history_when_bounded_fallback_is_complete_then_not } ), expected_payload={ - "app": "sqlbuild", - "v": 1, "run_id": "run-1", "resource_type": "model", "resource_name": "orders", @@ -290,7 +286,7 @@ def test_given_saturated_open_history_when_bounded_fallback_is_complete_then_not def test_given_sqlbuild_query_tag_when_parsing_then_context_is_returned( test_case: ParseQueryTagTestCase, ) -> None: - parsed: dict[str, Any] | None = _parse_sqlbuild_tag(test_case.query_tag) + parsed: dict[str, Any] | None = parse_sqlbuild_query_tag(test_case.query_tag) assert parsed == test_case.expected_payload @@ -319,7 +315,7 @@ def test_given_sqlbuild_query_tag_when_parsing_then_context_is_returned( def test_given_unrelated_or_invalid_query_tag_when_parsing_then_none_is_returned( test_case: ParseQueryTagTestCase, ) -> None: - assert _parse_sqlbuild_tag(test_case.query_tag) == test_case.expected_payload + assert parse_sqlbuild_query_tag(test_case.query_tag) == test_case.expected_payload @pytest.mark.parametrize( @@ -386,7 +382,7 @@ def test_given_snowflake_statement_when_classifying_then_compute_capability_is_c test_case: SnowflakeStatementClassificationTestCase, ) -> None: assert ( - _is_metadata_only_statement( + snowflake_statement_is_metadata_only( query_type=test_case.query_type, query_text=test_case.query_text, ) @@ -871,11 +867,11 @@ def test_given_retry_delay_exceeds_remaining_deadline_when_collecting_then_parti ) monotonic_values: Iterator[float] = iter((0.0, 0.0, 0.0, 0.0, 9.5)) monkeypatch.setattr( - "sqlbuild.cost._helpers.snowflake.time.monotonic", + "time.monotonic", lambda: next(monotonic_values), ) monkeypatch.setattr( - "sqlbuild.cost._helpers.snowflake.time.sleep", + "time.sleep", lambda seconds: pytest.fail(f"unexpected sleep for {seconds} seconds"), ) @@ -1168,7 +1164,6 @@ def test_given_visible_running_competitor_when_collecting_then_busy_time_is_shar def test_given_saturated_open_history_when_bounded_fallback_completes_then_running_share_is_retained( test_case: CollectSnowflakeCostTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: origin: datetime = datetime(2026, 8, 23, 10, tzinfo=UTC) ledger_path: Path = tmp_path / "statements.jsonl" @@ -1225,10 +1220,10 @@ def test_given_saturated_open_history_when_bounded_fallback_completes_then_runni connection: _SequencedConnection = _SequencedConnection( ((completed, running_competitor), (completed,)) ) - monkeypatch.setattr("sqlbuild.cost._helpers.snowflake._RESULT_LIMIT", 2) summary: RunCostSummary = collect_snowflake_cost( connection=connection, + result_limit=2, run_id="run-saturated-share", started_at=origin, completed_at=origin + timedelta(seconds=1), diff --git a/tests/unit/src/sqlbuild/cost/test_store.py b/tests/unit/src/sqlbuild/cost/test_store.py index b7ee3061e6..00fdc8ae77 100644 --- a/tests/unit/src/sqlbuild/cost/test_store.py +++ b/tests/unit/src/sqlbuild/cost/test_store.py @@ -6,8 +6,13 @@ import pytest -from sqlbuild._native import list_cost_runs, read_cost_run, resolve_cost_run, write_cost_run -from sqlbuild.cost.exceptions import CostArtifactError +from sqlbuild._native import ( + CostArtifactError, + list_cost_runs, + read_cost_run, + resolve_cost_run, + write_cost_run, +) from sqlbuild.cost.models import CostRunRecord from tests.unit.src.sqlbuild.cost._test_types import ( CostStoreTestCase, diff --git a/tests/unit/src/sqlbuild/diagnostics/test_process_resources.py b/tests/unit/src/sqlbuild/diagnostics/test_process_resources.py index 692793be69..3191ef6e1a 100644 --- a/tests/unit/src/sqlbuild/diagnostics/test_process_resources.py +++ b/tests/unit/src/sqlbuild/diagnostics/test_process_resources.py @@ -8,8 +8,8 @@ import pytest +from sqlbuild._native import ProcessResourceTracker from sqlbuild.diagnostics._helpers.process_resources import read_process_resources -from sqlbuild.diagnostics.classes.process_resource_tracker import ProcessResourceTracker from sqlbuild.diagnostics.models import ProcessResourceUsage from tests.unit.src.sqlbuild.diagnostics._test_types import ProcessResourceTestCase diff --git a/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py b/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py index a2dd5fd97d..1c26ec50b9 100644 --- a/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py +++ b/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py @@ -9,13 +9,10 @@ from sqlbuild._native import ( explicit_references_help, microbatch_guidance, + setting_help, + snippet_help, sql_analysis_opt_out_help, ) -from sqlbuild.errors.setting_help.main.join_helps import join_helps -from sqlbuild.errors.setting_help.main.model_header_help import model_header_help -from sqlbuild.errors.setting_help.main.setting_help import setting_help -from sqlbuild.errors.setting_help.main.setting_note import setting_note -from sqlbuild.errors.setting_help.main.snippet_help import snippet_help from tests.unit.src.sqlbuild.errors.setting_help.main._test_types import ( SettingCatalogueTestCase, SettingHelpTestCase, @@ -52,32 +49,15 @@ ' table_promotion_mode = "staged"', ), SettingHelpTestCase( - "an explicit note states the current value", - setting_note( + "an integer setting is bare", + setting_help( + purpose="to wait", file_name="sqlbuild_local.toml", section="settings", - key="sql_analysis", - value=False, - ), - "sqlbuild_local.toml sets [settings] sql_analysis = false", - ), - SettingHelpTestCase( - "a default note says the value is the default", - setting_note( - file_name="sqlbuild_project.toml", - section="references", - key="enforce_explicit", - value=True, - explicit=False, + key="timeout", + value=30, ), - "sqlbuild_project.toml does not set [references] enforce_explicit, " - "so it defaults to true", - ), - SettingHelpTestCase( - "a MODEL header help shows the exact entry", - model_header_help(purpose="to skip it", entry="sql_analysis false"), - "to skip it, add this to the MODEL header:\n MODEL (\n" - " sql_analysis false,\n ...\n );", + "to wait, set this in sqlbuild_local.toml:\n [settings]\n timeout = 30", ), SettingHelpTestCase( "a snippet help shows every line to write", @@ -90,11 +70,6 @@ " - name: product_types\n" " description: Product type catalogue", ), - SettingHelpTestCase( - "several helps keep one label each", - join_helps("first", "second"), - "first\n = help: second", - ), ), ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/executor/load/test_execute.py b/tests/unit/src/sqlbuild/executor/load/test_execute.py index 56e7a4cd4c..9e5203a0d5 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/load/test_execute.py @@ -9,10 +9,10 @@ import pytest +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.load.main._execute import execute_source_load from sqlbuild.executor.load.models import ( diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py index e3cb4245ec..21daaa917a 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py @@ -18,7 +18,12 @@ import pytest -from sqlbuild._native import AuditProgressReporter +from sqlbuild._native import ( + AuditProgressReporter, + CostContext, + read_statement_ledger, + record_statement, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import ( @@ -30,8 +35,6 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput -from sqlbuild.cost._helpers.ledger import read_statement_ledger, record_statement -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext, StatementLedgerEntry from sqlbuild.executor.auditing.main.resource_id import audit_resource_id from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py b/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py index 631389fa10..bcf529cd61 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py @@ -7,9 +7,9 @@ import pytest import sqlbuild.executor.pipeline.main.run as run_module +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams from sqlbuild.executor.build.types import BuildStatus diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index 5647bb96ac..df86a5e314 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any, ClassVar, cast -from sqlbuild._native import build_python_sql_run_lifecycle +from sqlbuild._native import CostContext, build_python_sql_run_lifecycle from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.assets import AssetContext, asset @@ -31,7 +31,6 @@ PythonSqlRunSelection, ) from sqlbuild.compiler.python_nodes.types import SkipMode -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py index 012182863a..e50de69cc8 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py @@ -7,10 +7,10 @@ import pytest +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.discovery.models import DiscoveredAssetFunction, DiscoveredTaskFunction from sqlbuild.compiler.python_nodes.types import PythonNodeStatus -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.node_results.classes.direct_store import DirectNodeResultStore from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py index 321a6da001..dc5da97064 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py @@ -9,6 +9,7 @@ import pytest from sqlbuild._native import ( + CostContext, ExecutionEventWriter, TerminalEventIndex, format_check_json, @@ -17,7 +18,6 @@ from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.node_results.models import NodeResultEnvelope from sqlbuild.executor.python_nodes._helpers.python_checks import execute_python_check_nodes diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py index cc01478c10..9c27ea3d76 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py @@ -10,11 +10,11 @@ from sqlbuild._native import cursor_render as render from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token +from sqlbuild._native import normalize_cursor_snapshot_grain from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner._helpers.resolve.cursor import normalize_cursor_snapshot_grain from sqlbuild.compiler.planner.exceptions import FutureCursorSafetyError from sqlbuild.compiler.planner.models import ( CursorBounds, diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py index 97043e06f8..934ce4aedc 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py @@ -7,12 +7,12 @@ import pytest +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks, render_hooks diff --git a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py index c20bee4cfb..1199b16010 100644 --- a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py +++ b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py @@ -8,8 +8,8 @@ import pytest +import sqlbuild._native as fixture_formatting_module import sqlbuild._native as native_module -import sqlbuild.compiler.planner._helpers.sql_tests.fixture_formatting as fixture_formatting_module from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs diff --git a/tests/unit/src/sqlbuild/presentation/classes/test_cli_document.py b/tests/unit/src/sqlbuild/presentation/classes/test_cli_document.py index 16e50370c2..b5ef96a5ff 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/test_cli_document.py +++ b/tests/unit/src/sqlbuild/presentation/classes/test_cli_document.py @@ -2,8 +2,7 @@ import pytest -from sqlbuild._native import CliDocument -from sqlbuild.presentation.classes.cli_style import CliStyle +from sqlbuild._native import CliDocument, CliStyle from tests.unit.src.sqlbuild.presentation.classes._test_types import CliDocumentTestCase diff --git a/tests/unit/src/sqlbuild/presentation/classes/test_cli_style.py b/tests/unit/src/sqlbuild/presentation/classes/test_cli_style.py index 54d7cc13f7..1658d74565 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/test_cli_style.py +++ b/tests/unit/src/sqlbuild/presentation/classes/test_cli_style.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.presentation.classes.cli_style import CliStyle +from sqlbuild._native import CliStyle from tests.unit.src.sqlbuild.presentation.classes._test_types import CliStyleTestCase diff --git a/tests/unit/src/sqlbuild/presentation/classes/test_transient_line_coordinator.py b/tests/unit/src/sqlbuild/presentation/classes/test_transient_line_coordinator.py index 8cf7e30bc6..42f18c13f2 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/test_transient_line_coordinator.py +++ b/tests/unit/src/sqlbuild/presentation/classes/test_transient_line_coordinator.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator +from sqlbuild._native import TransientLineCoordinator from tests.unit.src.sqlbuild.presentation.classes._test_types import ( TransientLineCoordinatorTestCase, ) diff --git a/tests/unit/src/sqlbuild/presentation/main/test_count_header.py b/tests/unit/src/sqlbuild/presentation/main/test_count_header.py index b3dde32fbe..93c3e01faf 100644 --- a/tests/unit/src/sqlbuild/presentation/main/test_count_header.py +++ b/tests/unit/src/sqlbuild/presentation/main/test_count_header.py @@ -2,8 +2,7 @@ import pytest -from sqlbuild._native import count_header_style -from sqlbuild.presentation.classes.cli_style import CliStyle +from sqlbuild._native import CliStyle, count_header_style from tests.unit.src.sqlbuild.presentation.main._test_types import CountHeaderTestCase diff --git a/tests/unit/src/sqlbuild/runtime/contracts/_test_types.py b/tests/unit/src/sqlbuild/runtime/contracts/_test_types.py index 5a55b6c11e..7b431a5856 100644 --- a/tests/unit/src/sqlbuild/runtime/contracts/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/contracts/_test_types.py @@ -16,7 +16,7 @@ class OpenConnectionFailureTestCase: description: str connection_config: dict[str, object] monotonic_times: tuple[float, float] - expected_error: Exception + expected_error: BaseException expected_event_order: tuple[str, ...] - expected_elapsed_seconds: float - expected_connection_count: int + expected_elapsed_seconds: tuple[float, ...] + expected_connection_counts: tuple[int, ...] diff --git a/tests/unit/src/sqlbuild/runtime/contracts/helpers.py b/tests/unit/src/sqlbuild/runtime/contracts/helpers.py index 36d233e974..d83ac1b730 100644 --- a/tests/unit/src/sqlbuild/runtime/contracts/helpers.py +++ b/tests/unit/src/sqlbuild/runtime/contracts/helpers.py @@ -29,7 +29,7 @@ def _execute(self, connection: Any, sql: str) -> object: class FailingConnectionAdapter(BaseAdapter): adapter_name: ClassVar[str] = "failing-connection-test" - def __init__(self, *, events: list[str], error: Exception) -> None: + def __init__(self, *, events: list[str], error: BaseException) -> None: self.events = events self.error = error self.connected_config: dict[str, object] | None = None diff --git a/tests/unit/src/sqlbuild/runtime/contracts/test_open_connection.py b/tests/unit/src/sqlbuild/runtime/contracts/test_open_connection.py index 79daaa59c6..5e8083d0e7 100644 --- a/tests/unit/src/sqlbuild/runtime/contracts/test_open_connection.py +++ b/tests/unit/src/sqlbuild/runtime/contracts/test_open_connection.py @@ -1,11 +1,11 @@ from __future__ import annotations +import time from collections.abc import Iterator import pytest -import sqlbuild.runtime.contracts.main.open_connection as open_connection_module -from sqlbuild.runtime.contracts.main.open_connection import open_connection_with_hooks +from sqlbuild._native import open_connection_with_hooks from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.unit.src.sqlbuild.runtime.contracts._test_types import ( OpenConnectionFailureTestCase, @@ -44,7 +44,7 @@ def test_given_connection_hooks_when_opening_succeeds_then_reports_order_and_ela connection=connection, ) monotonic_times: Iterator[float] = iter(test_case.monotonic_times) - monkeypatch.setattr(open_connection_module.time, "monotonic", lambda: next(monotonic_times)) + monkeypatch.setattr(time, "monotonic", lambda: next(monotonic_times)) def on_start(connection_count: int) -> None: events.append("start") @@ -80,9 +80,18 @@ def on_complete(connection_count: int, *, elapsed_seconds: int | float) -> None: monotonic_times=(20.0, 21.25), expected_error=RuntimeError("connection unavailable"), expected_event_order=("start", "connect", "error"), - expected_elapsed_seconds=1.25, - expected_connection_count=1, - ) + expected_elapsed_seconds=(1.25,), + expected_connection_counts=(1, 1), + ), + OpenConnectionFailureTestCase( + description="interrupted connection propagates without an error event", + connection_config={"database": "unavailable.duckdb"}, + monotonic_times=(20.0, 21.25), + expected_error=KeyboardInterrupt(), + expected_event_order=("start", "connect"), + expected_elapsed_seconds=(), + expected_connection_counts=(1,), + ), ], ids=lambda case: case.description, ) @@ -98,7 +107,7 @@ def test_given_connection_hooks_when_opening_fails_then_reports_error_and_rerais error=test_case.expected_error, ) monotonic_times: Iterator[float] = iter(test_case.monotonic_times) - monkeypatch.setattr(open_connection_module.time, "monotonic", lambda: next(monotonic_times)) + monkeypatch.setattr(time, "monotonic", lambda: next(monotonic_times)) def on_start(connection_count: int) -> None: events.append("start") @@ -122,5 +131,5 @@ def on_error(connection_count: int, *, elapsed_seconds: int | float) -> None: assert exc_info.value is test_case.expected_error assert adapter.connected_config is test_case.connection_config assert tuple(events) == test_case.expected_event_order - assert elapsed_seconds_values == [test_case.expected_elapsed_seconds] - assert connection_counts == [test_case.expected_connection_count] * 2 + assert tuple(elapsed_seconds_values) == test_case.expected_elapsed_seconds + assert tuple(connection_counts) == test_case.expected_connection_counts