diff --git a/Makefile b/Makefile index ce32a1d665..1882629b73 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SHELL := /bin/bash test-e2e-cold-compile-performance test-e2e-cache-compile-performance \ test-e2e-dense-compile-performance test-e2e-varied-cache-performance \ compiler-differential compiler-differential-dense \ - compiler-baselines + compiler-baselines planner-differential format: uv run ruff format . @@ -145,6 +145,7 @@ E2E_DUCKDB_CLI_PATHS := \ tests/e2e/scripts/cli_preview \ tests/e2e/scripts/compile_performance_ratio \ tests/e2e/scripts/compiler_differential \ + tests/e2e/scripts/planner_differential \ tests/e2e/scripts/release_performance \ tests/e2e/src/sqlbuild/cli/commands/main/adapters \ tests/e2e/src/sqlbuild/cli/commands/main/audit \ @@ -326,6 +327,17 @@ compiler-differential-dense: --corpus dense --dense-models $(COMPILER_DIFFERENTIAL_DENSE_MODELS) \ --jobs $(COMPILER_DIFFERENTIAL_JOBS) $(COMPILER_DIFFERENTIAL_ARGS) +# Stateful planner differential (see scripts/planner_differential/README.md): record a +# baseline on the tree before a planner change, then check the changed tree against it. +PLANNER_DIFFERENTIAL_MODE ?= check +PLANNER_DIFFERENTIAL_JOBS ?= 2 +PLANNER_DIFFERENTIAL_SEEDS ?= 6 +PLANNER_DIFFERENTIAL_ARGS ?= +planner-differential: + env PYTHONUNBUFFERED=1 uv run python -m scripts.run_planner_differential \ + $(PLANNER_DIFFERENTIAL_MODE) --jobs $(PLANNER_DIFFERENTIAL_JOBS) \ + --seeds $(PLANNER_DIFFERENTIAL_SEEDS) $(PLANNER_DIFFERENTIAL_ARGS) + skills: uv run python -m scripts.generate_docs_skill diff --git a/crates/sqlbuild-analysis/src/assembly/project/constants.rs b/crates/sqlbuild-analysis/src/assembly/project/constants.rs index cb4c75ce95..91b7a61a5e 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/constants.rs @@ -47,3 +47,16 @@ pub(crate) const AUDIT_TARGET_KIND_ENUM: &str = "AttachedAuditTargetKind"; pub(crate) const ENVIRONMENT_READ_PREFIX: &str = "ENV:"; /// Characters that end an `ENV:` name besides Python whitespace. pub(crate) const ENVIRONMENT_NAME_TERMINATORS: [char; 6] = ['(', ')', ',', '\'', '"', '}']; +/// The MODEL header opt-out help of SQL the parser rejects. +pub(crate) const MODEL_OPT_OUT_HELP: &str = "if this SQL is valid for your warehouse, skip SQL \ + analysis for this model, add this to the MODEL header:\n MODEL (\n \ + sql_analysis false,\n ...\n );\n and please report the SQL \ + so the parser can support it"; +/// The report-only help of rejected SQL that no model header owns. +pub(crate) const REPORT_ONLY_HELP: &str = + "if this SQL is valid for your warehouse, please report it so the parser can support it"; +/// The project-wide and single-run opt-out helps, each after its own help label. +pub(crate) const PROJECT_AND_RUN_OPT_OUT_HELP: &str = "\n = help: SQL analysis is on for this \ + project; to turn it off for every model, set this in sqlbuild_project.toml:\n \ + [settings]\n sql_analysis = false\n = help: to skip SQL analysis for one run, \ + use `--no-sql-analysis`"; diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs index 9ac75e428e..9deee5d6ea 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs @@ -3,4 +3,5 @@ pub mod check_sql_syntax; pub mod configured_namespace; pub mod environment_template_names; pub mod preserved_logical_namespace; +pub mod sql_analysis_opt_out_help; pub mod sql_syntax_error; diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/sql_analysis_opt_out_help.rs b/crates/sqlbuild-analysis/src/assembly/project/main/sql_analysis_opt_out_help.rs new file mode 100644 index 0000000000..eb807a897b --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/main/sql_analysis_opt_out_help.rs @@ -0,0 +1,16 @@ +//! Help for SQL the parser rejects: the exact header, project and single-run opt-outs. + +use crate::assembly::project::constants::{ + MODEL_OPT_OUT_HELP, PROJECT_AND_RUN_OPT_OUT_HELP, REPORT_ONLY_HELP, +}; + +/// The opt-out help; only SQL a model owns offers the MODEL header entry. +#[must_use] +pub fn sql_analysis_opt_out_help(model_owned: bool) -> String { + let first: &str = if model_owned { + MODEL_OPT_OUT_HELP + } else { + REPORT_ONLY_HELP + }; + format!("{first}{PROJECT_AND_RUN_OPT_OUT_HELP}") +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests.rs b/crates/sqlbuild-analysis/src/assembly/project/tests.rs index 3ac1cfe81c..73b4356612 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests.rs @@ -4,6 +4,8 @@ mod configured_namespace; mod environment_names; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_opt_out_help.rs"] +mod opt_out_help; #[path = "tests/test_project_assembly.rs"] mod project_assembly; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_opt_out_help.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_opt_out_help.rs new file mode 100644 index 0000000000..7c3d4884bd --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_opt_out_help.rs @@ -0,0 +1,34 @@ +use crate::assembly::project::main::sql_analysis_opt_out_help::sql_analysis_opt_out_help; +use crate::assembly::project::tests::test_types::OptOutHelpTestCase; + +#[test] +fn given_ownership_when_rendering_opt_out_help_then_matches_python_help() { + let test_cases = [ + OptOutHelpTestCase { + description: "model-owned SQL offers the MODEL header entry", + model_owned: true, + expected_help: "if this SQL is valid for your warehouse, skip SQL analysis for this \ + model, add this to the MODEL header:\n MODEL (\n \ + sql_analysis false,\n ...\n );\n and please \ + report the SQL so the parser can support it\n = help: SQL analysis is on for \ + this project; to turn it off for every model, set this in \ + sqlbuild_project.toml:\n [settings]\n sql_analysis = \ + false\n = help: to skip SQL analysis for one run, use `--no-sql-analysis`", + }, + OptOutHelpTestCase { + description: "other SQL only asks for a report", + model_owned: false, + expected_help: "if this SQL is valid for your warehouse, please report it so the \ + parser can support it\n = help: SQL analysis is on for this project; to turn \ + it off for every model, set this in sqlbuild_project.toml:\n \ + [settings]\n sql_analysis = false\n = help: to skip SQL analysis \ + for one run, use `--no-sql-analysis`", + }, + ]; + + for test_case in test_cases { + let help: String = sql_analysis_opt_out_help(test_case.model_owned); + + assert_eq!(help, test_case.expected_help, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs index 5d6acb6b71..5be1373d69 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs @@ -92,3 +92,9 @@ pub(crate) struct ConfiguredNamespaceTestCase { pub(crate) value: &'static str, pub(crate) expected_namespace: Option<&'static str>, } + +pub(crate) struct OptOutHelpTestCase { + pub(crate) description: &'static str, + pub(crate) model_owned: bool, + pub(crate) expected_help: &'static str, +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/map_expanded_offset.rs b/crates/sqlbuild-analysis/src/compiled_project/main/map_expanded_offset.rs new file mode 100644 index 0000000000..dcfda6c06d --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/map_expanded_offset.rs @@ -0,0 +1,10 @@ +//! Resolve an offset in expanded SQL back onto the authored body. + +use crate::compiled_project::_helpers::locating::map_through_passes; +use crate::compiled_project::models::SplicedSpan; + +/// The offset in the first pass's source, and whether an expansion generated it. +#[must_use] +pub fn map_expanded_offset(offset: usize, passes: &[Vec]) -> (usize, bool) { + map_through_passes(offset, passes) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/mod.rs b/crates/sqlbuild-analysis/src/compiled_project/main/mod.rs index c9a5f9143b..f3b2e21870 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/main/mod.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/main/mod.rs @@ -5,6 +5,7 @@ pub mod declaration_error_message; pub mod enum_member_literal; pub mod header_metadata_keys; pub mod located_scan_error; +pub mod map_expanded_offset; pub mod merge_call_site_references; pub mod model_facts; pub mod opt_out_position; diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index fc478207ea..383892f92e 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -19,6 +19,8 @@ pub mod lineage; pub mod lineage_command; pub mod lineage_graph; pub mod manifest; +pub mod planner_changes; +pub mod planner_strategy; pub mod python_plan; pub mod python_selection; pub mod python_values; diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/function_hashes.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/function_hashes.rs new file mode 100644 index 0000000000..d3d966b00b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/function_hashes.rs @@ -0,0 +1,47 @@ +//! Function hashes and identity payload equality of fingerprint metadata. + +use serde_json::Value; + +use crate::planner_changes::_helpers::json_payload::{decoded_object, python_equal}; +use crate::planner_changes::constants::LOCAL_FUNCTION_HASHES_METADATA_KEY; +use crate::planner_changes::models::IdentityPayload; + +/// Python equality of two identity payloads. +pub(crate) fn payloads_equal(left: &IdentityPayload, right: &IdentityPayload) -> bool { + match (left, right) { + (IdentityPayload::Object(left), IdentityPayload::Object(right)) => { + python_equal(&Value::Object(left.clone()), &Value::Object(right.clone())) + } + _ => left == right, + } +} + +/// Directly called functions whose recorded hash changed, sorted. +pub(crate) fn changed_local_function_names( + metadata_json: &str, + previous_metadata_json: &str, +) -> Vec { + let previous: Vec<(String, Value)> = local_function_hashes(previous_metadata_json); + let mut changed: Vec = Vec::new(); + for (name, hash) in local_function_hashes(metadata_json) { + let recorded: Option<&Value> = previous + .iter() + .find(|(previous_name, _)| *previous_name == name) + .map(|(_, previous_hash)| previous_hash); + if recorded.is_some_and(|recorded| !python_equal(recorded, &hash)) { + changed.push(name); + } + } + changed.sort(); + changed +} + +fn local_function_hashes(metadata_json: &str) -> Vec<(String, Value)> { + match decoded_object(metadata_json) { + Ok(Some(object)) => match object.get(LOCAL_FUNCTION_HASHES_METADATA_KEY) { + Some(Value::Object(hashes)) => hashes.clone().into_iter().collect(), + _ => Vec::new(), + }, + _ => Vec::new(), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/json_payload.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/json_payload.rs new file mode 100644 index 0000000000..6b7e1ec92b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/json_payload.rs @@ -0,0 +1,80 @@ +//! Decode fingerprint metadata as Python's `json.loads` does and write it as `json.dumps(sort_keys=True, separators=(",", ":"))`. + +use serde_json::{Map, Value}; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; + +/// The decoded top-level object, or None when the text is not a JSON object. +pub(crate) fn decoded_object(metadata_json: &str) -> Result>, ()> { + match serde_json::from_str::(metadata_json) { + Ok(Value::Object(object)) => Ok(Some(object)), + Ok(_) => Ok(None), + Err(_) => Err(()), + } +} + +/// `json.dumps(object, sort_keys=True, separators=(",", ":"), default=str)`. +pub(crate) fn canonical_text(object: Map) -> String { + let dialect: JsonDialect = JsonDialect::Stdlib(StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }); + dumps(&json_value(Value::Object(object)), &dialect).unwrap_or_default() +} + +/// Python equality of two decoded JSON values: numbers by value and booleans as 0 and 1. +pub(crate) fn python_equal(left: &Value, right: &Value) -> bool { + match (left, right) { + (Value::Array(left), Value::Array(right)) => { + left.len() == right.len() + && left + .iter() + .zip(right) + .all(|(left, right)| python_equal(left, right)) + } + (Value::Object(left), Value::Object(right)) => { + left.len() == right.len() + && left.iter().all(|(key, value)| { + right + .get(key) + .is_some_and(|other| python_equal(value, other)) + }) + } + (Value::Number(left_number), Value::Number(right_number)) + if !left_number.is_f64() && !right_number.is_f64() => + { + left_number == right_number + } + (Value::Number(_) | Value::Bool(_), Value::Number(_) | Value::Bool(_)) => { + numeric(left) == numeric(right) + } + _ => left == right, + } +} + +fn numeric(value: &Value) -> Option { + match value { + Value::Bool(flag) => Some(f64::from(u8::from(*flag))), + Value::Number(number) => number.as_f64(), + _ => None, + } +} + +fn json_value(value: Value) -> JsonValue { + match value { + Value::Null => JsonValue::Null, + Value::Bool(flag) => JsonValue::Bool(flag), + Value::Number(number) => match JsonInteger::parse(&number.to_string()) { + Some(integer) if !number.is_f64() => JsonValue::Integer(integer), + _ => JsonValue::Float(number.as_f64().unwrap_or_default()), + }, + Value::String(text) => JsonValue::String(text), + Value::Array(items) => JsonValue::Array(items.into_iter().map(json_value).collect()), + Value::Object(object) => JsonValue::Object( + object + .into_iter() + .map(|(key, value)| (key, json_value(value))) + .collect(), + ), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/mod.rs new file mode 100644 index 0000000000..f344a81462 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod function_hashes; +pub(crate) mod json_payload; +pub(crate) mod pick_more_aggressive; +pub(crate) mod rename_mappings; +pub(crate) mod schema_comparer; +pub(crate) mod with_origin_cursor_input_names; diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/pick_more_aggressive.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/pick_more_aggressive.rs new file mode 100644 index 0000000000..b41082c8ad --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/pick_more_aggressive.rs @@ -0,0 +1,20 @@ +//! Choose between two backfills. + +use crate::planner_changes::models::{Backfill, BackfillAction}; + +/// The more aggressive backfill (full over bounded over forward only); a tie keeps `first`. +pub(crate) fn pick_more_aggressive(first: Backfill, second: Backfill) -> Backfill { + if priority(first.action) >= priority(second.action) { + first + } else { + second + } +} + +fn priority(action: BackfillAction) -> u8 { + match action { + BackfillAction::ForwardOnly => 0, + BackfillAction::Bounded => 1, + BackfillAction::Full => 2, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/rename_mappings.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/rename_mappings.rs new file mode 100644 index 0000000000..b21175a866 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/rename_mappings.rs @@ -0,0 +1,63 @@ +//! Candidate rename mappings and their substituted queries. + +use crate::planner_changes::constants::MAX_CANDIDATE_MAPPINGS; +use crate::planner_changes::models::NameSpan; + +/// The name followed by every origin it was renamed from, breadth first. +pub(crate) fn lineage(name: &str, origins: &[(String, Vec)]) -> Vec { + let mut seen: Vec = vec![name.to_owned()]; + let mut index: usize = 0; + while index < seen.len() { + let current: String = seen[index].clone(); + if let Some((_, list)) = origins.iter().find(|(key, _)| *key == current) { + for origin in list { + if !seen.contains(origin) { + seen.push(origin.clone()); + } + } + } + index += 1; + } + seen +} + +/// `itertools.product(*choices)`, truncated to the first mappings tried. +pub(crate) fn product(choices: &[Vec]) -> Vec> { + let mut combinations: Vec> = vec![Vec::new()]; + for options in choices { + let mut next: Vec> = Vec::new(); + for prefix in &combinations { + for option in options { + if next.len() == MAX_CANDIDATE_MAPPINGS { + break; + } + let mut combination: Vec = prefix.clone(); + combination.push(option.clone()); + next.push(combination); + } + } + combinations = next; + } + combinations +} + +/// The query with each span replaced by its mapped origin name. +pub(crate) fn substitute( + characters: &[char], + spans: &[NameSpan], + mapping: &[(String, String)], +) -> String { + let mut text: String = String::new(); + let mut copied_to: usize = 0; + for span in spans { + text.extend(&characters[copied_to.min(span.start)..span.start]); + let replacement: &str = mapping + .iter() + .find(|(name, _)| *name == span.name) + .map_or(span.name.as_str(), |(_, origin)| origin.as_str()); + text.push_str(replacement); + copied_to = span.end; + } + text.extend(&characters[copied_to.min(characters.len())..]); + text +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/schema_comparer.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/schema_comparer.rs new file mode 100644 index 0000000000..0f5cff7051 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/schema_comparer.rs @@ -0,0 +1,159 @@ +//! Column-by-column schema comparison state. + +use std::collections::HashSet; + +use crate::planner_changes::models::{ + BackfillFacts, SchemaChangeKind, SchemaColumnSource, SchemaColumns, SchemaFindingRow, +}; + +/// Whether schema findings replay the policy: only alongside a definition, config or declared-column change. +pub(crate) fn schema_replays(facts: BackfillFacts) -> bool { + facts.schema_changed + && (facts.query_changed + || facts.functions_changed + || facts.config_changed + || facts.declared_columns_changed) +} + +/// Expected columns with optional types, and warehouse columns with dictionary semantics. +pub(crate) struct FoldedColumns { + pub(crate) yml: Vec<(String, Option)>, + pub(crate) inferred: Option)>>, + pub(crate) warehouse: Vec<(String, String)>, +} + +/// Lower-case names when the dialect folds them; later duplicate warehouse names replace earlier types in place. +pub(crate) fn folded_columns(columns: SchemaColumns, case_folding: bool) -> FoldedColumns { + let fold = |name: String| { + if case_folding { + name.to_lowercase() + } else { + name + } + }; + let yml: Vec<(String, Option)> = columns + .yml + .into_iter() + .map(|(name, kind)| (fold(name), Some(kind))) + .collect(); + let inferred: Option)>> = columns.inferred.map(|items| { + let mut folded: Vec<(String, Option)> = Vec::with_capacity(items.len()); + for (name, kind) in items { + folded.push((fold(name), kind)); + } + folded + }); + let mut warehouse: Vec<(String, String)> = Vec::new(); + for (name, kind) in columns.warehouse { + let name: String = fold(name); + match warehouse.iter_mut().find(|(existing, _)| *existing == name) { + Some(entry) => entry.1 = kind, + None => warehouse.push((name, kind)), + } + } + FoldedColumns { + yml, + inferred, + warehouse, + } +} + +/// Findings collected so far and the expected names already compared. +pub(crate) struct Comparer<'a, 'b, E> { + pub(crate) warehouse: &'a [(String, String)], + pub(crate) seen: HashSet, + pub(crate) findings: Vec, + pub(crate) types_equal: &'b mut dyn FnMut(&str, &str) -> Result, +} + +impl Comparer<'_, '_, E> { + /// Added and retyped columns; `skip_seen` skips names an earlier source already covered; then marks them seen. + pub(crate) fn compare( + &mut self, + expected: &[(String, Option)], + source: SchemaColumnSource, + skip_seen: bool, + ) -> Result<(), E> { + for (name, kind) in expected { + if skip_seen && self.seen.contains(name) { + continue; + } + match self.warehouse.iter().find(|(existing, _)| existing == name) { + None => self.findings.push(finding( + (SchemaChangeKind::ColumnAdded, name, source), + kind.clone(), + None, + )), + Some((_, actual)) => { + if let Some(expected_type) = kind + && !(self.types_equal)(actual, expected_type)? + { + self.findings.push(finding( + (SchemaChangeKind::ColumnTypeChanged, name, source), + Some(expected_type.clone()), + Some(actual.clone()), + )); + } + } + } + } + self.seen + .extend(expected.iter().map(|(name, _)| name.clone())); + Ok(()) + } + + /// Retyped warehouse columns that exactly one dynamic family accepts. + pub(crate) fn compare_families( + &mut self, + single_family_type: &mut dyn FnMut(&str) -> Result, E>, + ) -> Result<(), E> { + for (name, kind) in self.warehouse { + if self.seen.contains(name) { + continue; + } + let Some(family_type) = single_family_type(name)? else { + continue; + }; + self.seen.insert(name.clone()); + if !(self.types_equal)(&family_type, kind)? { + self.findings.push(finding( + ( + SchemaChangeKind::ColumnTypeChanged, + name, + SchemaColumnSource::Yml, + ), + Some(family_type), + Some(kind.clone()), + )); + } + } + Ok(()) + } + + /// Warehouse columns no expected column or family covered. + pub(crate) fn report_removed(&mut self, source: SchemaColumnSource) { + for (name, kind) in self.warehouse { + if !self.seen.contains(name) { + self.findings.push(finding( + (SchemaChangeKind::ColumnRemoved, name, source), + None, + Some(kind.clone()), + )); + } + } + } +} + +fn finding( + (kind, name, source): (SchemaChangeKind, &str, SchemaColumnSource), + expected_type: Option, + actual_type: Option, +) -> SchemaFindingRow { + SchemaFindingRow { + kind, + column_name: name.to_owned(), + source, + expected_type, + actual_type, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/with_origin_cursor_input_names.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/with_origin_cursor_input_names.rs new file mode 100644 index 0000000000..77bd0cc2bc --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/with_origin_cursor_input_names.rs @@ -0,0 +1,37 @@ +//! Name renamed cursor inputs by their origin models. + +use serde_json::{Map, Value}; + +use crate::planner_changes::_helpers::json_payload::{canonical_text, decoded_object}; +use crate::planner_changes::constants::{CONFIG_METADATA_KEY, CURSOR_INPUTS_CONFIG_KEY}; + +/// Canonical metadata JSON whose `config.cursor_inputs` keys use origin names; other text is returned unchanged. +pub(crate) fn with_origin_cursor_input_names( + metadata_json: &str, + renamed_refs: &[(String, String)], +) -> String { + let Ok(Some(mut object)) = decoded_object(metadata_json) else { + return metadata_json.to_owned(); + }; + let Some(Value::Object(config)) = object.get(CONFIG_METADATA_KEY) else { + return metadata_json.to_owned(); + }; + let Some(Value::Object(cursor_inputs)) = config.get(CURSOR_INPUTS_CONFIG_KEY) else { + return metadata_json.to_owned(); + }; + let mut origin_inputs: Map = Map::new(); + for (name, value) in cursor_inputs { + let origin: &str = renamed_refs + .iter() + .find(|(renamed, _)| renamed == name) + .map_or(name.as_str(), |(_, origin)| origin.as_str()); + origin_inputs.insert(origin.to_owned(), value.clone()); + } + let mut config: Map = config.clone(); + config.insert( + CURSOR_INPUTS_CONFIG_KEY.to_owned(), + Value::Object(origin_inputs), + ); + object.insert(CONFIG_METADATA_KEY.to_owned(), Value::Object(config)); + canonical_text(object) +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/constants.rs b/crates/sqlbuild-analysis/src/planner_changes/constants.rs new file mode 100644 index 0000000000..0ff28f52b8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/constants.rs @@ -0,0 +1,26 @@ +//! Planner change-detection constants shared with the Python planner. + +/// Fingerprint metadata key of the recorded audit gate. +pub const AUDIT_GATE_METADATA_KEY: &str = "audit_gate"; +/// Fingerprint metadata key of the migration fingerprint. +pub const MIGRATION_FINGERPRINT_METADATA_KEY: &str = "migration_fingerprint"; +/// Fingerprint metadata key of the declared-columns hash. +pub const DECLARED_COLUMNS_METADATA_KEY: &str = "declared_columns_hash"; +/// Fingerprint metadata key of directly called function hashes. +pub const LOCAL_FUNCTION_HASHES_METADATA_KEY: &str = "local_function_hashes"; +/// Fingerprint metadata key of the model config. +pub const CONFIG_METADATA_KEY: &str = "config"; +/// Model config key of cursor inputs. +pub const CURSOR_INPUTS_CONFIG_KEY: &str = "cursor_inputs"; +/// Metadata a fingerprint row without metadata records. +pub const EMPTY_FINGERPRINT_METADATA_JSON: &str = "{}"; +/// The replay policy value that replays nothing. +pub const FORWARD_ONLY_POLICY: &str = "forward"; +/// The replay policy value that replays everything. +pub const FULL_POLICY: &str = "full"; +/// Prefix of a bounded replay policy. +pub const BOUNDED_POLICY_PREFIX: &str = "bounded-"; +/// The dialect whose unquoted column names fold to lower case for comparison. +pub const CASE_FOLDING_DIALECT: &str = "snowflake"; +/// At most this many rename mappings are tried for one query. +pub const MAX_CANDIDATE_MAPPINGS: usize = 64; diff --git a/crates/sqlbuild-analysis/src/planner_changes/errors.rs b/crates/sqlbuild-analysis/src/planner_changes/errors.rs new file mode 100644 index 0000000000..7ba2438e8f --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/errors.rs @@ -0,0 +1,12 @@ +//! Planner change-detection errors. + +/// A `replay_on_change` value that is not a known policy. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct UnknownReplayPolicy(pub String); + +impl UnknownReplayPolicy { + /// The planner input error message. + pub fn message(&self) -> String { + format!("unknown replay_on_change '{}'", self.0) + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/detect_schema_changes.rs b/crates/sqlbuild-analysis/src/planner_changes/main/detect_schema_changes.rs new file mode 100644 index 0000000000..a0c52471c3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/detect_schema_changes.rs @@ -0,0 +1,45 @@ +//! Compare declared and inferred columns with the warehouse relation's columns. + +use std::collections::HashSet; + +use crate::planner_changes::_helpers::schema_comparer::{Comparer, FoldedColumns, folded_columns}; +use crate::planner_changes::models::{ + SchemaColumnSource, SchemaColumns, SchemaComparison, SchemaFindingRow, +}; +use crate::planner_changes::types::SchemaChecks; + +/// Findings in Python's order: declared and inferred columns, dynamic families, then removed columns. +pub fn detect_schema_changes( + columns: SchemaColumns, + comparison: SchemaComparison, + (types_equal, single_family_type): SchemaChecks<'_, E>, +) -> Result, E> { + let FoldedColumns { + yml, + inferred, + warehouse, + } = folded_columns(columns, comparison.case_folding); + let mut comparer = Comparer { + warehouse: &warehouse, + seen: HashSet::new(), + findings: Vec::new(), + types_equal, + }; + let inferred: &[(String, Option)] = inferred.as_deref().unwrap_or_default(); + if comparison.type_enforcement { + comparer.compare(&yml, SchemaColumnSource::Yml, false)?; + comparer.compare(inferred, SchemaColumnSource::SqlAnalysis, true)?; + } else { + comparer.compare(inferred, SchemaColumnSource::SqlAnalysis, true)?; + comparer.compare(&yml, SchemaColumnSource::Yml, true)?; + } + comparer.compare_families(single_family_type)?; + if comparison.type_enforcement || comparison.inferred_schema_complete { + comparer.report_removed(if yml.is_empty() { + SchemaColumnSource::SqlAnalysis + } else { + SchemaColumnSource::Yml + }); + } + Ok(comparer.findings) +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/metadata_changes.rs b/crates/sqlbuild-analysis/src/planner_changes/main/metadata_changes.rs new file mode 100644 index 0000000000..cd6a4ef57b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/metadata_changes.rs @@ -0,0 +1,37 @@ +//! Compare current and recorded version-identity metadata. + +use crate::planner_changes::_helpers::function_hashes::{ + changed_local_function_names, payloads_equal, +}; +use crate::planner_changes::_helpers::with_origin_cursor_input_names::with_origin_cursor_input_names; +use crate::planner_changes::main::version_identity_payload::version_identity_payload; +use crate::planner_changes::models::MetadataChanges; + +/// Called functions whose recorded hash changed (when queries are tracked) and whether the remaining identity changed. +pub fn metadata_changes( + (metadata_json, recorded_metadata_json): (&str, Option<&str>), + origin_names: Option<&[(String, String)]>, + query_change_tracking: bool, +) -> MetadataChanges { + let Some(recorded) = recorded_metadata_json else { + return MetadataChanges::default(); + }; + let changed_functions: Vec = if query_change_tracking { + changed_local_function_names(metadata_json, recorded) + } else { + Vec::new() + }; + let current: String = match origin_names { + Some(names) => with_origin_cursor_input_names(metadata_json, names), + None => metadata_json.to_owned(), + }; + let without_functions: bool = !changed_functions.is_empty(); + let config_changed: bool = !payloads_equal( + &version_identity_payload(Some(¤t), without_functions), + &version_identity_payload(Some(recorded), without_functions), + ); + MetadataChanges { + changed_functions, + config_changed, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/mod.rs b/crates/sqlbuild-analysis/src/planner_changes/main/mod.rs new file mode 100644 index 0000000000..86949d0c09 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/mod.rs @@ -0,0 +1,11 @@ +pub mod detect_schema_changes; +pub mod metadata_changes; +pub mod model_backfill; +pub mod model_change_kind; +pub mod origin_reference_names; +pub mod recorded_declared_columns_hash; +pub mod replay_policy_needed; +pub mod resolve_replay_on_change; +pub mod version_identity_payload; +pub mod with_declared_columns_hash; +pub mod without_declared_columns_hash; diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/model_backfill.rs b/crates/sqlbuild-analysis/src/planner_changes/main/model_backfill.rs new file mode 100644 index 0000000000..96878d4832 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/model_backfill.rs @@ -0,0 +1,27 @@ +//! Resolve the backfill of a detected change. + +use crate::planner_changes::_helpers::pick_more_aggressive::pick_more_aggressive; +use crate::planner_changes::_helpers::schema_comparer::schema_replays; +use crate::planner_changes::models::{Backfill, BackfillAction, BackfillFacts}; + +/// The replay policy applies to query or function changes, and to schema findings that come with any definition change; `replay` is the resolved policy, present when `replay_policy_needed`. +pub fn model_backfill(facts: BackfillFacts, replay: Option) -> Backfill { + let forward = Backfill { + action: BackfillAction::ForwardOnly, + duration: None, + }; + let Some(replay) = replay else { + return forward; + }; + let definition: Backfill = if facts.query_changed || facts.functions_changed { + replay.clone() + } else { + forward.clone() + }; + let schema: Backfill = if schema_replays(facts) { + replay + } else { + forward + }; + pick_more_aggressive(definition, schema) +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/model_change_kind.rs b/crates/sqlbuild-analysis/src/planner_changes/main/model_change_kind.rs new file mode 100644 index 0000000000..9e8f5cf00c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/model_change_kind.rs @@ -0,0 +1,18 @@ +//! Classify a model's change. + +use crate::planner_changes::models::{BackfillFacts, ChangeKind}; + +/// The change kind by precedence: query, then called functions, schema, config, else no change. +pub fn model_change_kind(facts: BackfillFacts) -> ChangeKind { + if facts.query_changed { + ChangeKind::QueryChanged + } else if facts.functions_changed { + ChangeKind::FunctionChanged + } else if facts.schema_changed { + ChangeKind::SchemaChanged + } else if facts.config_changed { + ChangeKind::ConfigChanged + } else { + ChangeKind::NoChange + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/origin_reference_names.rs b/crates/sqlbuild-analysis/src/planner_changes/main/origin_reference_names.rs new file mode 100644 index 0000000000..ab53e6e1ef --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/origin_reference_names.rs @@ -0,0 +1,54 @@ +//! Recognize a query whose only change is `__ref` names rewritten for renamed models. + +use std::collections::BTreeSet; + +use crate::planner_changes::_helpers::rename_mappings::{lineage, product, substitute}; +use crate::planner_changes::constants::MAX_CANDIDATE_MAPPINGS; +use crate::planner_changes::models::{NameSpan, RenameCandidate}; +use crate::planner_changes::types::RenameMatching; + +/// The first rename mapping (in name order, at most 64 tried) under which the query hashes to `recorded_hash`. +pub fn origin_reference_names( + (query_sql, recorded_hash): (&str, &str), + renames: &[RenameCandidate], + (find_spans, query_hash): RenameMatching<'_, E>, +) -> Result>, E> { + let mut origins: Vec<(String, Vec)> = Vec::new(); + for rename in renames.iter().filter(|rename| rename.applies) { + match origins + .iter_mut() + .find(|(name, _)| *name == rename.new_name) + { + Some((_, list)) => list.push(rename.origin_name.clone()), + None => origins.push((rename.new_name.clone(), vec![rename.origin_name.clone()])), + } + } + if origins.is_empty() { + return Ok(None); + } + let keys: BTreeSet = origins.iter().map(|(name, _)| name.clone()).collect(); + let spans: Vec = find_spans(&keys)?; + let names: Vec = spans + .iter() + .map(|span| span.name.clone()) + .collect::>() + .into_iter() + .collect(); + let choices: Vec> = names.iter().map(|name| lineage(name, &origins)).collect(); + let characters: Vec = query_sql.chars().collect(); + for combination in product(&choices).into_iter().take(MAX_CANDIDATE_MAPPINGS) { + let mapping: Vec<(String, String)> = names + .iter() + .zip(combination) + .filter(|(name, origin)| *name != origin) + .map(|(name, origin)| (name.clone(), origin)) + .collect(); + if mapping.is_empty() { + continue; + } + if query_hash(&substitute(&characters, &spans, &mapping))? == recorded_hash { + return Ok(Some(mapping)); + } + } + Ok(None) +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/recorded_declared_columns_hash.rs b/crates/sqlbuild-analysis/src/planner_changes/main/recorded_declared_columns_hash.rs new file mode 100644 index 0000000000..6a3fa0c963 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/recorded_declared_columns_hash.rs @@ -0,0 +1,17 @@ +//! Read the declared-columns hash a fingerprint recorded. + +use serde_json::Value; + +use crate::planner_changes::_helpers::json_payload::decoded_object; +use crate::planner_changes::constants::DECLARED_COLUMNS_METADATA_KEY; + +/// The recorded declared-columns hash, when the metadata holds one as a string. +pub fn recorded_declared_columns_hash(metadata_json: Option<&str>) -> Option { + match decoded_object(metadata_json?) { + Ok(Some(object)) => match object.get(DECLARED_COLUMNS_METADATA_KEY) { + Some(Value::String(hash)) => Some(hash.clone()), + _ => None, + }, + _ => None, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/replay_policy_needed.rs b/crates/sqlbuild-analysis/src/planner_changes/main/replay_policy_needed.rs new file mode 100644 index 0000000000..22471ad68c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/replay_policy_needed.rs @@ -0,0 +1,9 @@ +//! Whether a change reads the model's replay policy. + +use crate::planner_changes::_helpers::schema_comparer::schema_replays; +use crate::planner_changes::models::BackfillFacts; + +/// Query and function changes read `replay_on_change`, and so do schema findings that come with a definition change. +pub fn replay_policy_needed(facts: BackfillFacts) -> bool { + facts.query_changed || facts.functions_changed || schema_replays(facts) +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/resolve_replay_on_change.rs b/crates/sqlbuild-analysis/src/planner_changes/main/resolve_replay_on_change.rs new file mode 100644 index 0000000000..90ed3bf650 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/resolve_replay_on_change.rs @@ -0,0 +1,24 @@ +//! Parse a model's `replay_on_change` policy. + +use crate::planner_changes::constants::{BOUNDED_POLICY_PREFIX, FORWARD_ONLY_POLICY, FULL_POLICY}; +use crate::planner_changes::errors::UnknownReplayPolicy; +use crate::planner_changes::models::{Backfill, BackfillAction}; + +/// The backfill a detected version change replays: forward only, full, or bounded by a duration. +pub fn resolve_replay_on_change(raw: Option<&str>) -> Result { + let action = |action: BackfillAction, duration: Option| Backfill { action, duration }; + match raw { + None | Some(FORWARD_ONLY_POLICY) => Ok(action(BackfillAction::ForwardOnly, None)), + Some(FULL_POLICY) => Ok(action(BackfillAction::Full, None)), + Some(value) => match value + .strip_prefix(BOUNDED_POLICY_PREFIX) + .map(|duration| duration.strip_suffix('\n').unwrap_or(duration)) + { + Some(duration) if !duration.is_empty() && !duration.contains('\n') => Ok(action( + BackfillAction::Bounded, + Some(duration.trim().to_owned()), + )), + _ => Err(UnknownReplayPolicy(value.to_owned())), + }, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs b/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs new file mode 100644 index 0000000000..9bceac24a7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs @@ -0,0 +1,35 @@ +//! The fingerprint metadata that participates in version identity. + +use crate::planner_changes::_helpers::json_payload::decoded_object; +use crate::planner_changes::constants::{ + AUDIT_GATE_METADATA_KEY, DECLARED_COLUMNS_METADATA_KEY, LOCAL_FUNCTION_HASHES_METADATA_KEY, + MIGRATION_FINGERPRINT_METADATA_KEY, +}; +use crate::planner_changes::models::IdentityPayload; + +/// The metadata without audit-gate, migration and declared-column entries; with `without_functions`, also without function hashes. +pub fn version_identity_payload( + metadata_json: Option<&str>, + without_functions: bool, +) -> IdentityPayload { + let Some(text) = metadata_json else { + return IdentityPayload::Absent; + }; + match decoded_object(text) { + Err(()) => IdentityPayload::Raw(text.to_owned()), + Ok(None) => IdentityPayload::Absent, + Ok(Some(mut object)) => { + for key in [ + AUDIT_GATE_METADATA_KEY, + MIGRATION_FINGERPRINT_METADATA_KEY, + DECLARED_COLUMNS_METADATA_KEY, + ] { + object.shift_remove(key); + } + if without_functions { + object.shift_remove(LOCAL_FUNCTION_HASHES_METADATA_KEY); + } + IdentityPayload::Object(object) + } + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/with_declared_columns_hash.rs b/crates/sqlbuild-analysis/src/planner_changes/main/with_declared_columns_hash.rs new file mode 100644 index 0000000000..c3508647a9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/with_declared_columns_hash.rs @@ -0,0 +1,20 @@ +//! Record the declared-columns hash in fingerprint metadata. + +use serde_json::Value; + +use crate::planner_changes::_helpers::json_payload::{canonical_text, decoded_object}; +use crate::planner_changes::constants::DECLARED_COLUMNS_METADATA_KEY; + +/// Canonical metadata JSON that also records `declared_columns_hash`; text that is not an object is returned unchanged. +pub fn with_declared_columns_hash(metadata_json: &str, declared_columns_hash: &str) -> String { + match decoded_object(metadata_json) { + Ok(Some(mut object)) => { + object.insert( + DECLARED_COLUMNS_METADATA_KEY.to_owned(), + Value::String(declared_columns_hash.to_owned()), + ); + canonical_text(object) + } + _ => metadata_json.to_owned(), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/without_declared_columns_hash.rs b/crates/sqlbuild-analysis/src/planner_changes/main/without_declared_columns_hash.rs new file mode 100644 index 0000000000..a01da9c91e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/without_declared_columns_hash.rs @@ -0,0 +1,15 @@ +//! Drop the declared-columns hash from fingerprint metadata. + +use crate::planner_changes::_helpers::json_payload::{canonical_text, decoded_object}; +use crate::planner_changes::constants::DECLARED_COLUMNS_METADATA_KEY; + +/// Canonical metadata JSON without the declared-columns hash, or the text unchanged when it has none. +pub fn without_declared_columns_hash(metadata_json: &str) -> String { + match decoded_object(metadata_json) { + Ok(Some(mut object)) if object.contains_key(DECLARED_COLUMNS_METADATA_KEY) => { + object.shift_remove(DECLARED_COLUMNS_METADATA_KEY); + canonical_text(object) + } + _ => metadata_json.to_owned(), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/mod.rs b/crates/sqlbuild-analysis/src/planner_changes/mod.rs new file mode 100644 index 0000000000..9afa04ee22 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/mod.rs @@ -0,0 +1,11 @@ +//! Planner change detection: replay policy, version-identity metadata, schema findings, reference-rename origins and the per-model change decision. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/planner_changes/models.rs b/crates/sqlbuild-analysis/src/planner_changes/models.rs new file mode 100644 index 0000000000..71d7e90a84 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/models.rs @@ -0,0 +1,108 @@ +//! Planner change-detection inputs and results. + +/// The replay a change asks for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BackfillAction { + Full, + Bounded, + ForwardOnly, +} + +/// A backfill action with the duration of a bounded replay. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Backfill { + pub action: BackfillAction, + pub duration: Option, +} + +/// How a model changed since its recorded build, in precedence order. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ChangeKind { + QueryChanged, + FunctionChanged, + SchemaChanged, + ConfigChanged, + NoChange, +} + +/// What a schema finding reports about one column. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SchemaChangeKind { + ColumnAdded, + ColumnRemoved, + ColumnTypeChanged, +} + +/// Where an expected column was declared. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SchemaColumnSource { + Yml, + SqlAnalysis, +} + +/// One difference between expected and warehouse columns. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SchemaFindingRow { + pub kind: SchemaChangeKind, + pub column_name: String, + pub source: SchemaColumnSource, + pub expected_type: Option, + pub actual_type: Option, +} + +/// Expected and warehouse columns of one model: name and type. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SchemaColumns { + pub yml: Vec<(String, String)>, + pub inferred: Option)>>, + pub warehouse: Vec<(String, String)>, +} + +/// How a model's columns are compared. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct SchemaComparison { + pub type_enforcement: bool, + pub inferred_schema_complete: bool, + pub case_folding: bool, +} + +/// The parts of version-identity metadata that changed since the recorded build. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct MetadataChanges { + pub changed_functions: Vec, + pub config_changed: bool, +} + +/// Facts the backfill of a detected change depends on. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct BackfillFacts { + pub query_changed: bool, + pub functions_changed: bool, + pub config_changed: bool, + pub declared_columns_changed: bool, + pub schema_changed: bool, +} + +/// One model rename a reference may have been rewritten for; `applies` when the recorded build predates it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RenameCandidate { + pub new_name: String, + pub origin_name: String, + pub applies: bool, +} + +/// A `__ref` name in the query: start and end character offsets and the name. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NameSpan { + pub start: usize, + pub end: usize, + pub name: String, +} + +/// Fingerprint metadata as version identity compares it: absent, undecodable text, or the decoded object. +#[derive(Clone, Debug, PartialEq)] +pub enum IdentityPayload { + Absent, + Raw(String), + Object(serde_json::Map), +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/tests.rs b/crates/sqlbuild-analysis/src/planner_changes/tests.rs new file mode 100644 index 0000000000..949a25ebbd --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_planner_changes.rs"] +mod planner_changes; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_changes/tests/helpers.rs b/crates/sqlbuild-analysis/src/planner_changes/tests/helpers.rs new file mode 100644 index 0000000000..2412088ca4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/tests/helpers.rs @@ -0,0 +1,46 @@ +use crate::planner_changes::models::{ + Backfill, BackfillAction, NameSpan, SchemaChangeKind, SchemaColumnSource, SchemaFindingRow, +}; + +pub(super) fn backfill(action: BackfillAction, duration: Option<&str>) -> Backfill { + Backfill { + action, + duration: duration.map(str::to_owned), + } +} + +pub(super) fn finding( + (kind, name, source): (SchemaChangeKind, &str, SchemaColumnSource), + expected_type: Option<&str>, + actual_type: Option<&str>, +) -> SchemaFindingRow { + SchemaFindingRow { + kind, + column_name: name.to_owned(), + source, + expected_type: expected_type.map(str::to_owned), + actual_type: actual_type.map(str::to_owned), + } +} + +pub(super) fn pairs(items: &[(&str, &str)]) -> Vec<(String, String)> { + items + .iter() + .map(|(left, right)| ((*left).to_owned(), (*right).to_owned())) + .collect() +} + +/// Case-insensitive type equality, as a stand-in for the adapter's type normalization. +pub(super) fn types_equal(left: &str, right: &str) -> Result { + Ok(left.eq_ignore_ascii_case(right)) +} + +/// The span of the one quoted `name` in an ASCII query. +pub(super) fn quoted_span(query_sql: &str, name: &str) -> NameSpan { + let start: usize = query_sql.find(&format!("\"{name}\"")).unwrap_or_default() + 1; + NameSpan { + start, + end: start + name.len(), + name: name.to_owned(), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/tests/test_planner_changes.rs b/crates/sqlbuild-analysis/src/planner_changes/tests/test_planner_changes.rs new file mode 100644 index 0000000000..6e82bec30e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/tests/test_planner_changes.rs @@ -0,0 +1,427 @@ +use crate::planner_changes::main::detect_schema_changes::detect_schema_changes; +use crate::planner_changes::main::metadata_changes::metadata_changes; +use crate::planner_changes::main::model_backfill::model_backfill; +use crate::planner_changes::main::model_change_kind::model_change_kind; +use crate::planner_changes::main::origin_reference_names::origin_reference_names; +use crate::planner_changes::main::recorded_declared_columns_hash::recorded_declared_columns_hash; +use crate::planner_changes::main::replay_policy_needed::replay_policy_needed; +use crate::planner_changes::main::resolve_replay_on_change::resolve_replay_on_change; +use crate::planner_changes::main::with_declared_columns_hash::with_declared_columns_hash; +use crate::planner_changes::main::without_declared_columns_hash::without_declared_columns_hash; +use crate::planner_changes::models::{ + BackfillAction, BackfillFacts, ChangeKind, NameSpan, RenameCandidate, SchemaChangeKind, + SchemaColumnSource, SchemaColumns, SchemaComparison, +}; +use crate::planner_changes::tests::helpers::{backfill, finding, pairs, quoted_span, types_equal}; +use crate::planner_changes::tests::test_types::{ + DecisionTestCase, MetadataChangesTestCase, MetadataTextTestCase, OriginTestCase, + ReplayTestCase, SchemaTestCase, +}; + +#[test] +fn given_replay_policy_when_resolving_then_matches_python_parsing() { + let test_cases = [ + ReplayTestCase { + description: "absent", + raw: None, + expected_backfill: Ok(backfill(BackfillAction::ForwardOnly, None)), + }, + ReplayTestCase { + description: "forward", + raw: Some("forward"), + expected_backfill: Ok(backfill(BackfillAction::ForwardOnly, None)), + }, + ReplayTestCase { + description: "full", + raw: Some("full"), + expected_backfill: Ok(backfill(BackfillAction::Full, None)), + }, + ReplayTestCase { + description: "bounded with spaces", + raw: Some("bounded- 7d "), + expected_backfill: Ok(backfill(BackfillAction::Bounded, Some("7d"))), + }, + ReplayTestCase { + description: "bounded before a final newline", + raw: Some("bounded-3d\n"), + expected_backfill: Ok(backfill(BackfillAction::Bounded, Some("3d"))), + }, + ReplayTestCase { + description: "unknown", + raw: Some("forward_only"), + expected_backfill: Err("unknown replay_on_change 'forward_only'".to_owned()), + }, + ReplayTestCase { + description: "bounded without a duration", + raw: Some("bounded-"), + expected_backfill: Err("unknown replay_on_change 'bounded-'".to_owned()), + }, + ]; + for test_case in test_cases { + assert_eq!( + resolve_replay_on_change(test_case.raw).map_err(|error| error.message()), + test_case.expected_backfill, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_metadata_json_when_recording_declared_columns_then_writes_python_json() { + let test_cases = [ + MetadataTextTestCase { + description: "unsorted object with unicode and numbers", + metadata_json: r#"{"model_name": "orders", "config": {"b": 1.0, "a": "café"}, "n": 10}"#, + expected_with_hash: r#"{"config":{"a":"caf\u00e9","b":1.0},"declared_columns_hash":"h1","model_name":"orders","n":10}"#, + expected_without_hash: r#"{"model_name": "orders", "config": {"b": 1.0, "a": "café"}, "n": 10}"#, + expected_recorded_hash: None, + }, + MetadataTextTestCase { + description: "existing hash", + metadata_json: r#"{"declared_columns_hash":"h0","x":[1,2]}"#, + expected_with_hash: r#"{"declared_columns_hash":"h1","x":[1,2]}"#, + expected_without_hash: r#"{"x":[1,2]}"#, + expected_recorded_hash: Some("h0"), + }, + MetadataTextTestCase { + description: "not an object", + metadata_json: "[1]", + expected_with_hash: "[1]", + expected_without_hash: "[1]", + expected_recorded_hash: None, + }, + MetadataTextTestCase { + description: "not JSON", + metadata_json: "{oops", + expected_with_hash: "{oops", + expected_without_hash: "{oops", + expected_recorded_hash: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + ( + with_declared_columns_hash(test_case.metadata_json, "h1"), + without_declared_columns_hash(test_case.metadata_json), + recorded_declared_columns_hash(Some(test_case.metadata_json)), + ), + ( + test_case.expected_with_hash.to_owned(), + test_case.expected_without_hash.to_owned(), + test_case.expected_recorded_hash.map(str::to_owned), + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_recorded_metadata_when_comparing_then_reports_function_and_config_changes() { + let test_cases = [ + MetadataChangesTestCase { + description: "no recorded metadata", + current: r#"{"config":{"materialized":"table"}}"#, + recorded: None, + origin_names: None, + expected_functions: Vec::new(), + expected_config_changed: false, + }, + MetadataChangesTestCase { + description: "audit gate and declared columns are ignored, numbers compare by value", + current: r#"{"config":{"batch":1},"audit_gate":{"a":1},"declared_columns_hash":"x"}"#, + recorded: Some(r#"{"config":{"batch":1.0},"declared_columns_hash":"y"}"#), + origin_names: None, + expected_functions: Vec::new(), + expected_config_changed: false, + }, + MetadataChangesTestCase { + description: "config change", + current: r#"{"config":{"materialized":"view"}}"#, + recorded: Some(r#"{"config":{"materialized":"table"}}"#), + origin_names: None, + expected_functions: Vec::new(), + expected_config_changed: true, + }, + MetadataChangesTestCase { + description: "only function hashes changed", + current: r#"{"config":{},"local_function_hashes":{"zeta":"2","alpha":"2","new":"1"}}"#, + recorded: Some(r#"{"config":{},"local_function_hashes":{"zeta":"1","alpha":"1"}}"#), + origin_names: None, + expected_functions: vec!["alpha".to_owned(), "zeta".to_owned()], + expected_config_changed: false, + }, + MetadataChangesTestCase { + description: "renamed cursor input named by its origin", + current: r#"{"config":{"cursor_inputs":{"orders_v2":"ordered_at"}}}"#, + recorded: Some(r#"{"config":{"cursor_inputs":{"orders":"ordered_at"}}}"#), + origin_names: Some(pairs(&[("orders_v2", "orders")])), + expected_functions: Vec::new(), + expected_config_changed: false, + }, + ]; + for test_case in test_cases { + let changes = metadata_changes( + (test_case.current, test_case.recorded), + test_case.origin_names.as_deref(), + true, + ); + assert_eq!( + (changes.changed_functions, changes.config_changed), + ( + test_case.expected_functions, + test_case.expected_config_changed + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_change_facts_when_deciding_then_kind_and_backfill_follow_precedence() { + let test_cases = [ + DecisionTestCase { + description: "query change replays the policy", + facts: BackfillFacts { + query_changed: true, + schema_changed: true, + ..BackfillFacts::default() + }, + replay_on_change: Some("bounded-2d"), + expected_kind: ChangeKind::QueryChanged, + expected_backfill: backfill(BackfillAction::Bounded, Some("2d")), + }, + DecisionTestCase { + description: "schema drift alone stays forward", + facts: BackfillFacts { + schema_changed: true, + config_changed: false, + ..BackfillFacts::default() + }, + replay_on_change: Some("full"), + expected_kind: ChangeKind::SchemaChanged, + expected_backfill: backfill(BackfillAction::ForwardOnly, None), + }, + DecisionTestCase { + description: "schema findings with a config change replay", + facts: BackfillFacts { + schema_changed: true, + config_changed: true, + ..BackfillFacts::default() + }, + replay_on_change: Some("full"), + expected_kind: ChangeKind::SchemaChanged, + expected_backfill: backfill(BackfillAction::Full, None), + }, + DecisionTestCase { + description: "config change alone", + facts: BackfillFacts { + config_changed: true, + ..BackfillFacts::default() + }, + replay_on_change: Some("full"), + expected_kind: ChangeKind::ConfigChanged, + expected_backfill: backfill(BackfillAction::ForwardOnly, None), + }, + DecisionTestCase { + description: "function change", + facts: BackfillFacts { + functions_changed: true, + ..BackfillFacts::default() + }, + replay_on_change: None, + expected_kind: ChangeKind::FunctionChanged, + expected_backfill: backfill(BackfillAction::ForwardOnly, None), + }, + ]; + for test_case in test_cases { + assert_eq!( + ( + model_change_kind(test_case.facts), + model_backfill( + test_case.facts, + replay_policy_needed(test_case.facts) + .then(|| resolve_replay_on_change(test_case.replay_on_change).ok()) + .flatten(), + ) + ), + (test_case.expected_kind, test_case.expected_backfill), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_columns_when_comparing_schema_then_findings_keep_python_order() { + let test_cases = [ + SchemaTestCase { + description: "enforced declared columns, inferred extras and a removed column", + columns: SchemaColumns { + yml: pairs(&[("id", "INTEGER"), ("label", "TEXT")]), + inferred: Some(vec![ + ("id".to_owned(), Some("BIGINT".to_owned())), + ("amount".to_owned(), None), + ]), + warehouse: pairs(&[("id", "integer"), ("label", "INTEGER"), ("old", "TEXT")]), + }, + comparison: SchemaComparison { + type_enforcement: true, + inferred_schema_complete: false, + case_folding: false, + }, + expected_findings: vec![ + finding( + ( + SchemaChangeKind::ColumnTypeChanged, + "label", + SchemaColumnSource::Yml, + ), + Some("TEXT"), + Some("INTEGER"), + ), + finding( + ( + SchemaChangeKind::ColumnAdded, + "amount", + SchemaColumnSource::SqlAnalysis, + ), + None, + None, + ), + finding( + ( + SchemaChangeKind::ColumnRemoved, + "old", + SchemaColumnSource::Yml, + ), + None, + Some("TEXT"), + ), + ], + }, + SchemaTestCase { + description: "inferred columns first without enforcement, folded names", + columns: SchemaColumns { + yml: pairs(&[("ID", "INTEGER")]), + inferred: Some(vec![("Label".to_owned(), Some("TEXT".to_owned()))]), + warehouse: pairs(&[("id", "TEXT"), ("label", "TEXT"), ("extra_x", "INTEGER")]), + }, + comparison: SchemaComparison { + type_enforcement: false, + inferred_schema_complete: true, + case_folding: true, + }, + expected_findings: vec![ + finding( + ( + SchemaChangeKind::ColumnTypeChanged, + "id", + SchemaColumnSource::Yml, + ), + Some("INTEGER"), + Some("TEXT"), + ), + finding( + ( + SchemaChangeKind::ColumnTypeChanged, + "extra_x", + SchemaColumnSource::Yml, + ), + Some("TEXT"), + Some("INTEGER"), + ), + ], + }, + ]; + for test_case in test_cases { + let mut equal = types_equal; + let mut family = |name: &str| -> Result, ()> { + Ok(name.starts_with("extra_").then(|| "TEXT".to_owned())) + }; + let findings = detect_schema_changes( + test_case.columns, + test_case.comparison, + (&mut equal, &mut family), + ); + assert_eq!( + findings, + Ok(test_case.expected_findings), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_renamed_references_when_matching_then_returns_origin_mapping() { + let test_cases = [ + OriginTestCase { + description: "one renamed reference", + query_sql: r#"SELECT * FROM __ref("orders_v2") JOIN __ref("customers")"#, + recorded_query: r#"SELECT * FROM __ref("orders") JOIN __ref("customers")"#, + renames: vec![("orders_v2", "orders", true)], + span_names: vec!["orders_v2"], + expected_mapping: Some(pairs(&[("orders_v2", "orders")])), + }, + OriginTestCase { + description: "chained renames reach the oldest name", + query_sql: r#"SELECT * FROM __ref("orders_v3")"#, + recorded_query: r#"SELECT * FROM __ref("orders")"#, + renames: vec![ + ("orders_v3", "orders_v2", true), + ("orders_v2", "orders", true), + ], + span_names: vec!["orders_v3"], + expected_mapping: Some(pairs(&[("orders_v3", "orders")])), + }, + OriginTestCase { + description: "rename recorded after the build does not apply", + query_sql: r#"SELECT * FROM __ref("orders_v2")"#, + recorded_query: r#"SELECT * FROM __ref("orders")"#, + renames: vec![("orders_v2", "orders", false)], + span_names: vec!["orders_v2"], + expected_mapping: None, + }, + OriginTestCase { + description: "other edits are not a rename", + query_sql: r#"SELECT id FROM __ref("orders_v2")"#, + recorded_query: r#"SELECT * FROM __ref("orders")"#, + renames: vec![("orders_v2", "orders", true)], + span_names: vec!["orders_v2"], + expected_mapping: None, + }, + ]; + for test_case in test_cases { + let renames: Vec = test_case + .renames + .iter() + .map(|(new_name, origin_name, applies)| RenameCandidate { + new_name: (*new_name).to_owned(), + origin_name: (*origin_name).to_owned(), + applies: *applies, + }) + .collect(); + let found: Vec = test_case + .span_names + .iter() + .map(|name| quoted_span(test_case.query_sql, name)) + .collect(); + let mut spans = |_: &std::collections::BTreeSet| -> Result, ()> { + Ok(found.clone()) + }; + let mut hash = |text: &str| -> Result { Ok(text.to_owned()) }; + assert_eq!( + origin_reference_names( + (test_case.query_sql, test_case.recorded_query), + &renames, + (&mut spans, &mut hash), + ), + Ok(test_case.expected_mapping), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_changes/tests/test_types.rs new file mode 100644 index 0000000000..0074d77cc8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/tests/test_types.rs @@ -0,0 +1,50 @@ +use crate::planner_changes::models::{ + Backfill, BackfillFacts, ChangeKind, SchemaColumns, SchemaComparison, SchemaFindingRow, +}; + +pub(super) struct ReplayTestCase { + pub(super) description: &'static str, + pub(super) raw: Option<&'static str>, + pub(super) expected_backfill: Result, +} + +pub(super) struct MetadataTextTestCase { + pub(super) description: &'static str, + pub(super) metadata_json: &'static str, + pub(super) expected_with_hash: &'static str, + pub(super) expected_without_hash: &'static str, + pub(super) expected_recorded_hash: Option<&'static str>, +} + +pub(super) struct MetadataChangesTestCase { + pub(super) description: &'static str, + pub(super) current: &'static str, + pub(super) recorded: Option<&'static str>, + pub(super) origin_names: Option>, + pub(super) expected_functions: Vec, + pub(super) expected_config_changed: bool, +} + +pub(super) struct DecisionTestCase { + pub(super) description: &'static str, + pub(super) facts: BackfillFacts, + pub(super) replay_on_change: Option<&'static str>, + pub(super) expected_kind: ChangeKind, + pub(super) expected_backfill: Backfill, +} + +pub(super) struct SchemaTestCase { + pub(super) description: &'static str, + pub(super) columns: SchemaColumns, + pub(super) comparison: SchemaComparison, + pub(super) expected_findings: Vec, +} + +pub(super) struct OriginTestCase { + pub(super) description: &'static str, + pub(super) query_sql: &'static str, + pub(super) recorded_query: &'static str, + pub(super) renames: Vec<(&'static str, &'static str, bool)>, + pub(super) span_names: Vec<&'static str>, + pub(super) expected_mapping: Option>, +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/types.rs b/crates/sqlbuild-analysis/src/planner_changes/types.rs new file mode 100644 index 0000000000..64b3278afe --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/types.rs @@ -0,0 +1,23 @@ +//! Planner change-detection callback types. + +use std::collections::BTreeSet; + +use crate::planner_changes::models::NameSpan; + +/// Finds the `__ref` spans of some names. +pub type SpanFinder<'a, E> = dyn FnMut(&BTreeSet) -> Result, E> + 'a; + +/// Whether two type strings are equal (warehouse or family type first, expected type second). +pub type TypesEqual<'a, E> = dyn FnMut(&str, &str) -> Result + 'a; + +/// The type of the one dynamic family accepting a column name, if exactly one does. +pub type SingleFamilyType<'a, E> = dyn FnMut(&str) -> Result, E> + 'a; + +/// Hashes a candidate query. +pub type QueryHasher<'a, E> = dyn FnMut(&str) -> Result + 'a; + +/// The column checks schema comparison asks the caller: type equality and single-family type. +pub type SchemaChecks<'a, E> = (&'a mut TypesEqual<'a, E>, &'a mut SingleFamilyType<'a, E>); + +/// How rename matching finds `__ref` spans and hashes a candidate query. +pub type RenameMatching<'a, E> = (&'a mut SpanFinder<'a, E>, &'a mut QueryHasher<'a, E>); diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/decimal_scale.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/decimal_scale.rs new file mode 100644 index 0000000000..c10f3ec64d --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/decimal_scale.rs @@ -0,0 +1,36 @@ +//! The scale of a decimal type written as `DECIMAL`, `NUMERIC` or `NUMBER`. + +use crate::planner_strategy::constants::DECIMAL_TYPE_NAMES; + +/// Digits after optional spaces, then optional spaces: the rest of the text. +fn digits_then_spaces(text: &str) -> Option<(&str, &str)> { + let text: &str = text.trim_start_matches(char::is_whitespace); + let end: usize = text + .find(|character: char| !character.is_ascii_digit()) + .unwrap_or(text.len()); + if end == 0 { + return None; + } + Some(( + &text[..end], + text[end..].trim_start_matches(char::is_whitespace), + )) +} + +/// Whether a full `DECIMAL[(p[, s])]` match has scale zero (absent counts as zero); `None` when the text does not match. +pub(crate) fn decimal_scale_is_zero(normalized: &str) -> Option { + let rest: &str = DECIMAL_TYPE_NAMES + .iter() + .find_map(|name| normalized.strip_prefix(name))?; + if rest.is_empty() { + return Some(true); + } + let inner: &str = rest.strip_prefix('(')?.strip_suffix(')')?; + let (_, after_precision) = digits_then_spaces(inner)?; + if after_precision.is_empty() { + return Some(true); + } + let (scale, tail) = digits_then_spaces(after_precision.strip_prefix(',')?)?; + tail.is_empty() + .then(|| scale.chars().all(|digit| digit == '0')) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/lineage_walk.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/lineage_walk.rs new file mode 100644 index 0000000000..04424283af --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/lineage_walk.rs @@ -0,0 +1,42 @@ +//! Breadth-first walks over SQL reference lineage. + +use std::collections::{HashSet, VecDeque}; + +use crate::planner_strategy::constants::{FUNCTION_REFERENCE_KINDS, MODEL_REFERENCE_KIND}; +use crate::planner_strategy::models::ReferenceIdentity; +use crate::planner_strategy::types::ReferenceGraph; + +/// The references a model or function reference reads, when the project has its target. +pub(crate) fn upstream_references( + graph: &G, + identity: &ReferenceIdentity, +) -> Result, G::Error> { + let references: Option> = if identity.kind == MODEL_REFERENCE_KIND { + graph.model_references(&identity.name)? + } else if FUNCTION_REFERENCE_KINDS.contains(&identity.kind.as_str()) { + graph.function_references(&identity.name)? + } else { + None + }; + Ok(references.unwrap_or_default()) +} + +/// A first-in, first-out queue of references that skips identities it has expanded. +pub(crate) struct LineageQueue { + pub(crate) pending: VecDeque, + pub(crate) visited: HashSet, +} + +impl LineageQueue { + pub(crate) fn new(roots: Vec) -> Self { + Self { + pending: roots.into(), + visited: HashSet::new(), + } + } + + /// Whether an identity is newly visited. + pub(crate) fn visit(&mut self, identity: &ReferenceIdentity) -> bool { + self.visited.insert(identity.clone()) + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/mod.rs new file mode 100644 index 0000000000..2aa0d6e824 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod decimal_scale; +pub(crate) mod lineage_walk; +pub(crate) mod python_int; +pub(crate) mod reasons; +pub(crate) mod schema_policy; +pub(crate) mod timeline; +pub(crate) mod type_names; +pub(crate) mod warnings; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/python_int.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/python_int.rs new file mode 100644 index 0000000000..b93cf5529e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/python_int.rs @@ -0,0 +1,31 @@ +//! Python's `int(text)` for batch sizes. + +use sqlbuild_core::text::main::decimal_digit_value::decimal_digit_value; + +/// Base-10 `int(text)`: surrounding whitespace, a sign, decimal digits and single underscores between them; `None` where Python raises or the value leaves `i128`. +pub(crate) fn python_int(text: &str) -> Option { + let trimmed: &str = text.trim(); + let (negative, digits) = match trimmed.chars().next() { + Some('-') => (true, &trimmed[1..]), + Some('+') => (false, &trimmed[1..]), + _ => (false, trimmed), + }; + let mut value: i128 = 0; + let mut previous_underscore: bool = true; + for character in digits.chars() { + if character == '_' { + if previous_underscore { + return None; + } + previous_underscore = true; + continue; + } + let digit: u32 = decimal_digit_value(character)?; + value = value.checked_mul(10)?.checked_add(i128::from(digit))?; + previous_underscore = false; + } + if previous_underscore { + return None; + } + Some(if negative { -value } else { value }) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/reasons.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/reasons.rs new file mode 100644 index 0000000000..01e1e5f61b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/reasons.rs @@ -0,0 +1,30 @@ +//! Plan reasons of a model's own change. + +use crate::planner_strategy::models::{ChangeFacts, PlanChangeKind, PlanReason}; + +/// The reason of a first run or a definition change; `None` for renames and no change. +pub(crate) fn own_change_reason(kind: PlanChangeKind) -> Option { + match kind { + PlanChangeKind::FirstRun => Some(PlanReason::FirstRun), + PlanChangeKind::QueryChanged => Some(PlanReason::QueryChanged), + PlanChangeKind::FunctionChanged => Some(PlanReason::FunctionChanged), + PlanChangeKind::ConfigChanged => Some(PlanReason::ConfigChanged), + PlanChangeKind::SchemaChanged => Some(PlanReason::SchemaChanged), + PlanChangeKind::Renamed | PlanChangeKind::NoChange => None, + } +} + +/// The reason a full backfill rebuilds the table. +pub(crate) fn backfill_reason(change: &ChangeFacts) -> PlanReason { + if change.query_changed { + PlanReason::QueryChanged + } else if !change.changed_functions.is_empty() { + PlanReason::FunctionChanged + } else if change.config_changed { + PlanReason::ConfigChanged + } else if !change.schema_findings.is_empty() { + PlanReason::SchemaChanged + } else { + PlanReason::FullRefresh + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/schema_policy.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/schema_policy.rs new file mode 100644 index 0000000000..ffd98fbd0e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/schema_policy.rs @@ -0,0 +1,11 @@ +//! The effective `on_schema_change` policy. + +use crate::planner_strategy::constants::APPEND_NEW_COLUMNS; + +/// `on_schema_change or append_new_columns`. +pub(crate) fn effective_on_schema_change(on_schema_change: Option<&str>) -> &str { + match on_schema_change { + Some(value) if !value.is_empty() => value, + _ => APPEND_NEW_COLUMNS, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/timeline.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/timeline.rs new file mode 100644 index 0000000000..6fc532bdd2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/timeline.rs @@ -0,0 +1,17 @@ +//! One clipped batch step on a caller's timeline. + +use crate::planner_strategy::types::Timeline; + +/// `min(moment + step, end)`, keeping `moment + step` on ties as Python's `min` does. +pub(crate) fn clipped_step( + timeline: &T, + moment: &T::Moment, + end: &T::Moment, +) -> Result { + let next: T::Moment = timeline.advance(moment)?; + Ok(if timeline.precedes(end, &next)? { + end.clone() + } else { + next + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs new file mode 100644 index 0000000000..aae5b546d0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs @@ -0,0 +1,27 @@ +//! Cursor types of warehouse type names and Polyglot type arguments. + +use serde_json::{Map, Value}; + +use crate::planner_strategy::constants::{INTEGER_TYPE_NAMES, TIMESTAMP_TYPE_NAMES}; +use crate::planner_strategy::models::CursorType; + +/// The cursor type of a known type name. +pub(crate) fn named_type(name: &str) -> Option { + if TIMESTAMP_TYPE_NAMES.contains(&name) { + return Some(CursorType::Timestamp); + } + INTEGER_TYPE_NAMES + .contains(&name) + .then_some(CursorType::Integer) +} + +/// Python's `str()` of one parsed type argument, upper-cased. +pub(crate) fn argument(args: &Map, key: &str) -> String { + match args.get(key) { + None => String::new(), + Some(Value::String(text)) => text.to_uppercase(), + Some(Value::Null) => "NONE".to_owned(), + Some(Value::Bool(flag)) => if *flag { "TRUE" } else { "FALSE" }.to_owned(), + Some(other) => other.to_string().to_uppercase(), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/warnings.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/warnings.rs new file mode 100644 index 0000000000..303e0b0bfd --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/warnings.rs @@ -0,0 +1,90 @@ +//! The schema-finding and forward-only replay warnings of one model. + +use crate::planner_changes::models::{SchemaChangeKind, SchemaColumnSource, SchemaFindingRow}; +use crate::planner_strategy::constants::QUERY_CHANGE_TRACKING_HELP; +use crate::planner_strategy::models::{ + ChangeFacts, MaterializationType, WarningInputs, WarningRow, WarningSeverity, +}; + +/// Python's `str()` of an optional type. +fn shown(value: Option<&String>) -> &str { + value.map_or("None", String::as_str) +} + +pub(crate) fn warning(severity: WarningSeverity, message: String) -> WarningRow { + WarningRow { + severity, + message, + code: None, + } +} + +/// The warning of one schema finding, if it has one. +pub(crate) fn finding_warning( + finding: &SchemaFindingRow, + type_enforcement: bool, +) -> Option { + let name: &str = &finding.column_name; + let expected: &str = shown(finding.expected_type.as_ref()); + let actual: &str = shown(finding.actual_type.as_ref()); + if type_enforcement && finding.source == SchemaColumnSource::Yml { + return (finding.kind == SchemaChangeKind::ColumnTypeChanged).then(|| { + warning( + WarningSeverity::Warning, + format!( + "enforced column {name} type mismatch: expected {expected}, warehouse has {actual}" + ), + ) + }); + } + if finding.source != SchemaColumnSource::SqlAnalysis { + return None; + } + let description: String = match finding.kind { + SchemaChangeKind::ColumnAdded => match finding.expected_type.as_deref() { + Some(expected) if !expected.is_empty() => { + format!("new column not in warehouse ({expected})") + } + _ => "new column not in warehouse".to_owned(), + }, + SchemaChangeKind::ColumnTypeChanged => { + format!("expected {expected}, warehouse has {actual}") + } + SchemaChangeKind::ColumnRemoved => return None, + }; + Some(warning( + WarningSeverity::Info, + format!("inferred column {name}: {description}"), + )) +} + +/// The forward-only replay warning of an incremental model whose definition changed. +pub(crate) fn forward_only_warning( + inputs: &WarningInputs, + change: &ChangeFacts, +) -> Option { + if inputs.materialization != MaterializationType::Incremental + || !(change.query_changed || !change.changed_functions.is_empty()) + || !change.backfill_forward_only + { + return None; + } + let model_name: &str = &inputs.model_name; + let subject: String = if change.query_changed { + "the model's SQL changed".to_owned() + } else { + format!("function {} changed", change.changed_functions.join(", ")) + }; + Some(warning( + WarningSeverity::Warning, + format!( + "{model_name}: {subject}, so already-built dates keep their old results - only new \ + dates will use the new query. To apply the change to existing data, choose how much \ + to reprocess: replay_on_change full (rebuild every date with the new SQL), \ + replay_on_change bounded- e.g. bounded-14d (reprocess just that window), \ + or leave it forward-only (existing dates as-is, new dates use the new query - the \ + default). If this model's output does not depend on the SQL that changed, this is \ + safe to ignore. {QUERY_CHANGE_TRACKING_HELP}" + ), + )) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/constants.rs b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs new file mode 100644 index 0000000000..9029fe4ae4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs @@ -0,0 +1,78 @@ +//! Planner strategy constants shared with the Python planner. + +use crate::planner_strategy::models::PlanAction; + +/// Error code of an incremental model without a strategy. +pub const MISSING_STRATEGY_ERROR_CODE: &str = "S201"; +/// Error code of an incremental model with an unknown strategy. +pub const UNKNOWN_STRATEGY_ERROR_CODE: &str = "S202"; +/// Error code of a watermark input outside the model's lineage. +pub const LINEAGE_INPUT_ERROR_CODE: &str = "S302"; +/// Warning code of a recorded relation that no longer exists. +pub const RECORDED_RELATION_MISSING_WARNING_CODE: &str = "RECORDED_RELATION_MISSING"; +/// Warning code of a model planning more batches than its limit. +pub const MICROBATCH_LIMIT_WARNING_CODE: &str = "MICROBATCH_LIMIT_EXCEEDED"; +/// The `on_schema_change` value of a model that leaves schema changes alone. +pub const IGNORE_SCHEMA_CHANGE: &str = "ignore"; +/// The `on_schema_change` value of a model that fails on schema changes. +pub const FAIL_SCHEMA_CHANGE: &str = "fail"; +/// The `on_schema_change` value that adds new columns; the default. +pub const APPEND_NEW_COLUMNS: &str = "append_new_columns"; +/// The `on_schema_change` value that adds, drops and retypes columns. +pub const SYNC_ALL_COLUMNS: &str = "sync_all_columns"; +/// Incremental strategies and the plan action of each. +pub const INCREMENTAL_STRATEGIES: [(&str, PlanAction); 3] = [ + ("append", PlanAction::IncrementalAppend), + ("delete_insert", PlanAction::IncrementalDeleteInsert), + ("merge", PlanAction::IncrementalMerge), +]; +/// How to silence forward-only change warnings. +pub const QUERY_CHANGE_TRACKING_HELP: &str = "To silence these warnings, set this in sqlbuild_project.toml:\n [settings]\n query_change_tracking = false"; +/// Reference kind of a model reference. +pub const MODEL_REFERENCE_KIND: &str = "ref"; +/// Reference kind of a source reference. +pub const SOURCE_REFERENCE_KIND: &str = "source"; +/// Reference kinds of SQL function calls. +pub const FUNCTION_REFERENCE_KINDS: [&str; 2] = ["udf", "table_fn"]; +/// Warehouse type names classified as timestamp cursors. +pub const TIMESTAMP_TYPE_NAMES: [&str; 13] = [ + "TIMESTAMP", + "TIMESTAMPNTZ", + "TIMESTAMPLTZ", + "TIMESTAMPTZ", + "TIMESTAMP_S", + "TIMESTAMP_MS", + "TIMESTAMP_NS", + "DATE", + "DATE32", + "DATETIME", + "DATETIME2", + "DATETIME64", + "SMALLDATETIME", +]; +/// Warehouse type names classified as integer cursors. +pub const INTEGER_TYPE_NAMES: [&str; 17] = [ + "INT", + "BIG_INT", + "BIGINT", + "SMALL_INT", + "SMALLINT", + "TINY_INT", + "TINYINT", + "MEDIUMINT", + "INT128", + "INT256", + "UINT", + "UBIGINT", + "USMALLINT", + "UTINYINT", + "UMEDIUMINT", + "UINT128", + "UINT256", +]; +/// Decimal type names whose zero-scale form is an integer cursor. +pub const DECIMAL_TYPE_NAMES: [&str; 3] = ["DECIMAL", "NUMERIC", "NUMBER"]; +/// The type Polyglot reports for a type it does not know. +pub const POLYGLOT_CUSTOM_TYPE_NAME: &str = "CUSTOM"; +/// The dialect cursor type classification parses unknown types with. +pub const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/errors.rs b/crates/sqlbuild-analysis/src/planner_strategy/errors.rs new file mode 100644 index 0000000000..2be7a8dd5b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/errors.rs @@ -0,0 +1,92 @@ +//! Planner strategy errors. + +use crate::planner_strategy::constants::{ + LINEAGE_INPUT_ERROR_CODE, MISSING_STRATEGY_ERROR_CODE, UNKNOWN_STRATEGY_ERROR_CODE, +}; + +/// An incremental model whose strategy is missing or unknown. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StrategyError { + Missing { model: String }, + Unknown { model: String, strategy: String }, +} + +impl StrategyError { + /// The planner input error message. + pub fn message(&self) -> String { + match self { + Self::Missing { model } => { + format!("incremental model '{model}' is missing required incremental_strategy") + } + Self::Unknown { model, strategy } => { + format!("incremental model '{model}' has unknown strategy '{strategy}'") + } + } + } + + /// The planner input error code. + pub fn code(&self) -> &'static str { + match self { + Self::Missing { .. } => MISSING_STRATEGY_ERROR_CODE, + Self::Unknown { .. } => UNKNOWN_STRATEGY_ERROR_CODE, + } + } +} + +/// A lineage walk that failed in the caller or did not find a watermark input. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum LineageFailure { + Graph(E), + NotInLineage { model: String, input: String }, +} + +impl LineageFailure { + /// The planner input error message of a missing input. + pub fn message(model: &str, input: &str) -> String { + format!( + "model '{model}': cursor_inputs watermark references '{input}', but it is not in the \ + model's upstream lineage" + ) + } + + /// The planner input error code of a missing input. + pub fn code() -> &'static str { + LINEAGE_INPUT_ERROR_CODE + } +} + +/// A watermark input served by a capped microbatch producer. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CappedWatermarkInput { + pub model: String, + pub input: String, + pub producer: String, +} + +impl CappedWatermarkInput { + /// The compile input error message. + pub fn message(&self) -> String { + let Self { + model, + input, + producer, + } = self; + if producer != input { + return format!( + "model '{model}' uses watermark input '{input}' derived from capped producer \ + '{producer}'; capped producers cannot serve as watermark inputs" + ); + } + format!( + "model '{model}' uses capped producer '{producer}' as a watermark input; capped \ + producers cannot serve as watermark inputs" + ) + } +} + +/// A warehouse type Polyglot could not parse, which classifies as no cursor type. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TypeParseFailure { + pub normalized_type: String, + pub message: String, +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/cap_integer_bounds.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/cap_integer_bounds.rs new file mode 100644 index 0000000000..f792d81e90 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/cap_integer_bounds.rs @@ -0,0 +1,20 @@ +//! An integer cursor range capped to its first or last batches. + +use crate::planner_strategy::_helpers::python_int::python_int; +use crate::planner_strategy::models::{CapDirection, CappedBound}; + +/// The moved bound, or `None` when the batch size is not a positive integer. +pub fn cap_integer_bounds( + (start, end): (i128, i128), + batch_size: &str, + max_batches: i128, + direction: CapDirection, +) -> Option> { + let size: i128 = python_int(batch_size).filter(|size| *size > 0)?; + if direction == CapDirection::FromStart { + return Some(CappedBound::End(end.min(start + size * max_batches))); + } + let batch_count: i128 = (end - start + size - 1).div_euclid(size).max(0); + let skipped: i128 = (batch_count - max_batches).max(0); + Some(CappedBound::Start(start + size * skipped)) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/cap_timeline_bounds.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/cap_timeline_bounds.rs new file mode 100644 index 0000000000..c5fa4065d6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/cap_timeline_bounds.rs @@ -0,0 +1,33 @@ +//! A timestamp cursor range capped to its first or last batches. + +use crate::planner_strategy::_helpers::timeline::clipped_step; +use crate::planner_strategy::models::{CapDirection, CappedBound}; +use crate::planner_strategy::types::Timeline; + +/// The new end after `max_batches` steps from the start, or the start of the last `max_batches` batches. +pub fn cap_timeline_bounds( + timeline: &T, + (start, end): (T::Moment, T::Moment), + max_batches: i128, + direction: CapDirection, +) -> Result, T::Error> { + if direction == CapDirection::FromStart { + let mut capped_end: T::Moment = start; + for _ in 0..max_batches.max(0) { + capped_end = clipped_step(timeline, &capped_end, &end)?; + } + return Ok(CappedBound::End(capped_end)); + } + let mut boundaries: Vec = vec![start]; + while let Some(last) = boundaries.last().cloned() { + if !timeline.precedes(&last, &end)? { + break; + } + boundaries.push(clipped_step(timeline, &last, &end)?); + } + let last_index: usize = boundaries.len().saturating_sub(1); + let kept: usize = usize::try_from(max_batches).unwrap_or(0).min(last_index); + Ok(CappedBound::Start( + boundaries.swap_remove(last_index - kept), + )) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/capped_watermark_input.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/capped_watermark_input.rs new file mode 100644 index 0000000000..ec4a447e31 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/capped_watermark_input.rs @@ -0,0 +1,71 @@ +//! Watermark inputs served by capped microbatch producers. + +use crate::planner_strategy::_helpers::lineage_walk::{LineageQueue, upstream_references}; +use crate::planner_strategy::constants::MODEL_REFERENCE_KIND; +use crate::planner_strategy::errors::{CappedWatermarkInput, LineageFailure}; +use crate::planner_strategy::main::lineage_reference::lineage_reference; +use crate::planner_strategy::models::{ProducerFacts, ReferenceIdentity}; +use crate::planner_strategy::types::ReferenceGraph; + +/// The first capped microbatch model upstream of a reference, following microbatch producers through their watermark inputs. +fn capped_producer( + graph: &G, + reference: G::Reference, +) -> Result, LineageFailure> { + let mut queue: LineageQueue = LineageQueue::new(vec![reference]); + while let Some(current) = queue.pending.pop_front() { + let identity: ReferenceIdentity = + graph.identity(¤t).map_err(LineageFailure::Graph)?; + if !queue.visit(&identity) { + continue; + } + if identity.kind == MODEL_REFERENCE_KIND { + let Some(roots) = graph + .model_references(&identity.name) + .map_err(LineageFailure::Graph)? + else { + continue; + }; + let facts: ProducerFacts = graph + .producer_facts(&identity.name) + .map_err(LineageFailure::Graph)?; + if facts.microbatch && facts.capped { + return Ok(Some(identity.name)); + } + if facts.microbatch { + for input in graph + .watermark_inputs(&identity.name) + .map_err(LineageFailure::Graph)? + { + let next: G::Reference = + lineage_reference(graph, (&identity.name, roots.clone()), &input)?; + queue.pending.push_back(next); + } + continue; + } + } + queue + .pending + .extend(upstream_references(graph, &identity).map_err(LineageFailure::Graph)?); + } + Ok(None) +} + +/// Python's `validate_capped_watermark_inputs` for a microbatch model with these references and watermark inputs: the first input a capped producer serves. +pub fn capped_watermark_input( + graph: &G, + (model, roots): (&str, Vec), + inputs: Vec, +) -> Result, LineageFailure> { + for input in inputs { + let reference: G::Reference = lineage_reference(graph, (model, roots.clone()), &input)?; + if let Some(producer) = capped_producer(graph, reference)? { + return Ok(Some(CappedWatermarkInput { + model: model.to_owned(), + input, + producer, + })); + } + } + Ok(None) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs new file mode 100644 index 0000000000..22dedf16bd --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs @@ -0,0 +1,41 @@ +//! The cursor type a warehouse SQL type supports. + +use serde_json::{Map, Value}; + +use crate::planner_strategy::_helpers::decimal_scale::decimal_scale_is_zero; +use crate::planner_strategy::_helpers::type_names::{argument, named_type}; +use crate::planner_strategy::constants::{GENERIC_DIALECT, POLYGLOT_CUSTOM_TYPE_NAME}; +use crate::planner_strategy::errors::TypeParseFailure; +use crate::planner_strategy::models::CursorType; +use crate::type_system::main::polyglot_type_args::polyglot_type_args; + +/// Timestamp or integer for supported cross-adapter types; `Err` when Polyglot cannot parse an unknown type. +pub fn classify_cursor_sql_type(sql_type: &str) -> Result, TypeParseFailure> { + let normalized: String = sql_type + .to_uppercase() + .split_whitespace() + .collect::>() + .join(" "); + if let Some(integral) = decimal_scale_is_zero(&normalized) { + return Ok(integral.then_some(CursorType::Integer)); + } + let base: String = normalized + .split('(') + .next() + .unwrap_or_default() + .trim() + .replace(' ', "_"); + if let Some(cursor_type) = named_type(&base) { + return Ok(Some(cursor_type)); + } + let args: Map = + polyglot_type_args(&normalized, GENERIC_DIALECT).map_err(|message| TypeParseFailure { + normalized_type: normalized.clone(), + message, + })?; + let mut type_name: String = argument(&args, "data_type"); + if type_name == POLYGLOT_CUSTOM_TYPE_NAME { + type_name = argument(&args, "name").replace('_', ""); + } + Ok(named_type(&type_name)) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/count_integer_batches.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/count_integer_batches.rs new file mode 100644 index 0000000000..f8ce1417aa --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/count_integer_batches.rs @@ -0,0 +1,21 @@ +//! The number of batches in an integer cursor range. + +use crate::planner_strategy::_helpers::python_int::python_int; + +/// Batches of `batch_size` from `start` up to `end`; equal bounds are one batch when asked. +pub fn count_integer_batches( + (start, end): (i128, i128), + batch_size: &str, + equal_bounds_are_batch: bool, +) -> i128 { + let Some(size) = python_int(batch_size) else { + return 0; + }; + if equal_bounds_are_batch && start == end { + return 1; + } + if size <= 0 || start >= end { + return 0; + } + (end - start + size - 1).div_euclid(size) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/count_timeline_batches.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/count_timeline_batches.rs new file mode 100644 index 0000000000..38981643f8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/count_timeline_batches.rs @@ -0,0 +1,25 @@ +//! The number of batches between two aligned moments. + +use crate::planner_strategy::_helpers::timeline::clipped_step; +use crate::planner_strategy::types::Timeline; + +/// Steps from `start` until `end`, the last one clipped; equal bounds are one batch when asked. +pub fn count_timeline_batches( + timeline: &T, + (start, end): (T::Moment, T::Moment), + equal_bounds_are_batch: bool, +) -> Result { + if equal_bounds_are_batch + && !timeline.precedes(&start, &end)? + && !timeline.precedes(&end, &start)? + { + return Ok(1); + } + let mut current: T::Moment = start; + let mut count: i128 = 0; + while timeline.precedes(¤t, &end)? { + current = clipped_step(timeline, ¤t, &end)?; + count += 1; + } + Ok(count) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs new file mode 100644 index 0000000000..d7b05b6719 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs @@ -0,0 +1,60 @@ +//! Whether a cursor column's warehouse type matches the declared cursor type. + +use crate::planner_strategy::errors::TypeParseFailure; +use crate::planner_strategy::main::classify_cursor_sql_type::classify_cursor_sql_type; +use crate::planner_strategy::models::{CursorType, CursorTypeCheck, WarningRow, WarningSeverity}; + +fn cursor_type_name(cursor_type: CursorType) -> &'static str { + match cursor_type { + CursorType::Timestamp => "timestamp", + CursorType::Integer => "integer", + } +} + +/// An error (with SQL analysis) or warning when the warehouse type is the other cursor type, and any Polyglot parse error that left it unclassified. +pub fn cursor_type_warning( + check: &CursorTypeCheck, +) -> (Option, Option) { + let (Some(column), Some(declared)) = (&check.cursor_column, &check.cursor_type) else { + return (None, None); + }; + let Some((_, warehouse_type)) = check + .warehouse_columns + .iter() + .find(|(name, _)| name.to_uppercase() == column.to_uppercase()) + else { + return (None, None); + }; + let declared: CursorType = match declared.as_str() { + "timestamp" => CursorType::Timestamp, + "integer" => CursorType::Integer, + _ => return (None, None), + }; + let detected: CursorType = match classify_cursor_sql_type(warehouse_type) { + Ok(Some(detected)) => detected, + Ok(None) => return (None, None), + Err(failure) => return (None, Some(failure)), + }; + if detected == declared { + return (None, None); + } + let (severity, verb) = if check.sql_analysis_enabled { + (WarningSeverity::Error, "is") + } else { + (WarningSeverity::Warning, "appears to be") + }; + let message: String = format!( + "cursor column '{column}' has warehouse type '{warehouse_type}' which {verb} {}, but \ + cursor_type is '{}'", + cursor_type_name(detected), + cursor_type_name(declared) + ); + ( + Some(WarningRow { + severity, + message, + code: None, + }), + None, + ) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/effective_run_scope.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/effective_run_scope.rs new file mode 100644 index 0000000000..9ffdb9128d --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/effective_run_scope.rs @@ -0,0 +1,17 @@ +//! The run scope an audit gets after degradation. + +use crate::planner_strategy::models::AuditRunScope; + +/// Delta-and-final scope only applies to audits attached to incremental or snapshot models. +pub fn effective_run_scope( + requested: AuditRunScope, + attached_materialization: Option<&str>, +) -> AuditRunScope { + if requested == AuditRunScope::Final { + return AuditRunScope::Final; + } + match attached_materialization { + Some("incremental" | "snapshot") => AuditRunScope::DeltaAndFinal, + _ => AuditRunScope::Final, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/lineage_reference.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/lineage_reference.rs new file mode 100644 index 0000000000..f3a7564272 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/lineage_reference.rs @@ -0,0 +1,32 @@ +//! The reference a watermark input names in a model's upstream lineage. + +use crate::planner_strategy::_helpers::lineage_walk::{LineageQueue, upstream_references}; +use crate::planner_strategy::errors::LineageFailure; +use crate::planner_strategy::models::ReferenceIdentity; +use crate::planner_strategy::types::ReferenceGraph; + +/// The first reference named `input`, breadth first from a model's references through model and SQL function references. +pub fn lineage_reference( + graph: &G, + (model, roots): (&str, Vec), + input: &str, +) -> Result> { + let mut queue: LineageQueue = LineageQueue::new(roots); + while let Some(reference) = queue.pending.pop_front() { + let identity: ReferenceIdentity = + graph.identity(&reference).map_err(LineageFailure::Graph)?; + if identity.name == input { + return Ok(reference); + } + if !queue.visit(&identity) { + continue; + } + queue + .pending + .extend(upstream_references(graph, &identity).map_err(LineageFailure::Graph)?); + } + Err(LineageFailure::NotInLineage { + model: model.to_owned(), + input: input.to_owned(), + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs new file mode 100644 index 0000000000..f45163514c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs @@ -0,0 +1,15 @@ +//! The materialization type of a model's `materialized` value. + +use crate::planner_strategy::models::MaterializationType; + +/// A table when unset; an unknown string is a custom materialization. +pub fn materialization_type(materialized: Option<&str>) -> MaterializationType { + match materialized { + None | Some("table") => MaterializationType::Table, + Some("view") => MaterializationType::View, + Some("incremental") => MaterializationType::Incremental, + Some("snapshot") => MaterializationType::Snapshot, + Some("seed") => MaterializationType::Seed, + Some(_) => MaterializationType::Custom, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/microbatch_limit_warning.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/microbatch_limit_warning.rs new file mode 100644 index 0000000000..c7b6dd3371 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/microbatch_limit_warning.rs @@ -0,0 +1,23 @@ +//! The message of a model planning more batches than its limit. + +/// `None` within the limit; `cap` says whether the action caps (rather than fails or warns). +pub fn microbatch_limit_warning( + model_name: &str, + (max_batches, batch_count): (Option, i128), + (action, cap): (&str, bool), +) -> Option { + let max_batches: i128 = max_batches.filter(|maximum| batch_count > *maximum)?; + let follow_up: String = if cap { + format!( + "Selected {max_batches} batches and deferred {}.", + batch_count - max_batches + ) + } else { + "Use --max-microbatches with an intentional invocation-specific value for this backfill." + .to_owned() + }; + Some(format!( + "MICROBATCH LIMIT EXCEEDED: model '{model_name}' planned {batch_count} batches, above \ + the per-model limit of {max_batches} (action={action}). {follow_up}" + )) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs new file mode 100644 index 0000000000..533eb51d7b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs @@ -0,0 +1,15 @@ +pub mod cap_integer_bounds; +pub mod cap_timeline_bounds; +pub mod capped_watermark_input; +pub mod classify_cursor_sql_type; +pub mod count_integer_batches; +pub mod count_timeline_batches; +pub mod cursor_type_warning; +pub mod effective_run_scope; +pub mod lineage_reference; +pub mod materialization_type; +pub mod microbatch_limit_warning; +pub mod model_plan_action; +pub mod model_warnings; +pub mod schema_actions; +pub mod singular_attachment; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs new file mode 100644 index 0000000000..437d66d599 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs @@ -0,0 +1,87 @@ +//! The plan action and reason of one model. + +use crate::planner_strategy::_helpers::reasons::{backfill_reason, own_change_reason}; +use crate::planner_strategy::constants::INCREMENTAL_STRATEGIES; +use crate::planner_strategy::errors::StrategyError; +use crate::planner_strategy::models::{ + ChangeFacts, MaterializationType, ModelStrategy, PlanAction, PlanChangeKind, PlanReason, +}; + +/// Python's `resolve_model_plan_action`: disabled models skip, then materialization decides. +pub fn model_plan_action( + model: &ModelStrategy, + change: &ChangeFacts, + full_refresh: bool, +) -> Result<(PlanAction, PlanReason), StrategyError> { + if model.disabled { + return Ok((PlanAction::Skip, PlanReason::Disabled)); + } + let refreshed = |fallback: PlanReason, renamed: bool| -> PlanReason { + if full_refresh { + return PlanReason::FullRefresh; + } + if renamed && change.kind == PlanChangeKind::Renamed { + return PlanReason::Renamed; + } + own_change_reason(change.kind).unwrap_or(fallback) + }; + match model.materialization { + MaterializationType::Custom => { + return Ok((PlanAction::Custom, refreshed(PlanReason::NoChange, false))); + } + MaterializationType::View => { + return Ok(( + PlanAction::CreateView, + refreshed(PlanReason::NoChange, true), + )); + } + MaterializationType::Snapshot => { + return Ok(( + PlanAction::Snapshot, + refreshed(PlanReason::NormalIncremental, false), + )); + } + MaterializationType::Table + | MaterializationType::Incremental + | MaterializationType::Seed => {} + } + if full_refresh { + return Ok((PlanAction::CreateTable, PlanReason::FullRefresh)); + } + if change.kind == PlanChangeKind::FirstRun { + return Ok((PlanAction::CreateTable, PlanReason::FirstRun)); + } + if change.kind == PlanChangeKind::Renamed { + return Ok((PlanAction::CreateTable, PlanReason::Renamed)); + } + if change.backfill_full { + return Ok((PlanAction::CreateTable, backfill_reason(change))); + } + let edited: Option = match change.kind { + PlanChangeKind::QueryChanged => Some(PlanReason::QueryChanged), + PlanChangeKind::FunctionChanged => Some(PlanReason::FunctionChanged), + PlanChangeKind::ConfigChanged => Some(PlanReason::ConfigChanged), + PlanChangeKind::SchemaChanged => Some(PlanReason::SchemaChanged), + PlanChangeKind::FirstRun | PlanChangeKind::Renamed | PlanChangeKind::NoChange => None, + }; + if model.materialization == MaterializationType::Table { + return Ok(( + PlanAction::CreateTable, + edited.unwrap_or(PlanReason::NoChange), + )); + } + let Some(strategy) = model.incremental_strategy.as_deref() else { + return Err(StrategyError::Missing { + model: model.name.clone(), + }); + }; + let reason: PlanReason = edited.unwrap_or(PlanReason::NormalIncremental); + INCREMENTAL_STRATEGIES + .iter() + .find(|(name, _)| *name == strategy) + .map(|(_, action)| (*action, reason)) + .ok_or_else(|| StrategyError::Unknown { + model: model.name.clone(), + strategy: strategy.to_owned(), + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/model_warnings.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/model_warnings.rs new file mode 100644 index 0000000000..4a32320b83 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/model_warnings.rs @@ -0,0 +1,53 @@ +//! The plan warnings of one model. + +use crate::planner_changes::models::SchemaChangeKind; +use crate::planner_strategy::_helpers::schema_policy::effective_on_schema_change; +use crate::planner_strategy::_helpers::warnings::{finding_warning, forward_only_warning, warning}; +use crate::planner_strategy::constants::{ + FAIL_SCHEMA_CHANGE, IGNORE_SCHEMA_CHANGE, RECORDED_RELATION_MISSING_WARNING_CODE, +}; +use crate::planner_strategy::models::{ChangeFacts, WarningInputs, WarningRow, WarningSeverity}; + +/// Python's `build_model_warnings`, in its order. +pub fn model_warnings(inputs: &WarningInputs, change: &ChangeFacts) -> Vec { + let effective: &str = effective_on_schema_change(inputs.on_schema_change.as_deref()); + let mut warnings: Vec = Vec::new(); + if change.recorded_build_relation_missing { + warnings.push(WarningRow { + severity: WarningSeverity::Warning, + message: format!( + "{}: SQLBuild state records a previous build, but the relation no longer exists \ + in the warehouse. It may have been dropped outside SQLBuild; planning a first \ + run to recreate it.", + inputs.model_name + ), + code: Some(RECORDED_RELATION_MISSING_WARNING_CODE), + }); + } + if effective == FAIL_SCHEMA_CHANGE && !change.schema_findings.is_empty() { + warnings.push(warning( + WarningSeverity::Error, + "schema change detected and on_schema_change is set to fail".to_owned(), + )); + } + warnings.extend( + change + .schema_findings + .iter() + .filter_map(|finding| finding_warning(finding, inputs.type_enforcement)), + ); + warnings.extend(forward_only_warning(inputs, change)); + let backfill_findings: bool = change.schema_findings.iter().any(|finding| { + matches!( + finding.kind, + SchemaChangeKind::ColumnAdded | SchemaChangeKind::ColumnTypeChanged + ) + }); + if backfill_findings && !inputs.has_schema_actions && effective == IGNORE_SCHEMA_CHANGE { + warnings.push(warning( + WarningSeverity::Info, + "schema change detected but on_schema_change is set to ignore".to_owned(), + )); + } + warnings +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/schema_actions.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/schema_actions.rs new file mode 100644 index 0000000000..88c7b50b55 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/schema_actions.rs @@ -0,0 +1,41 @@ +//! Schema actions resolved from a model's schema findings and `on_schema_change`. + +use crate::planner_changes::models::{SchemaChangeKind, SchemaFindingRow}; +use crate::planner_strategy::_helpers::schema_policy::effective_on_schema_change; +use crate::planner_strategy::constants::{APPEND_NEW_COLUMNS, SYNC_ALL_COLUMNS}; +use crate::planner_strategy::models::{SchemaActionKind, SchemaActionRow}; + +/// Added columns under `append_new_columns`; added, removed and retyped columns under `sync_all_columns`. +pub fn schema_actions( + findings: &[SchemaFindingRow], + on_schema_change: Option<&str>, +) -> Vec { + let effective: &str = effective_on_schema_change(on_schema_change); + let sync: bool = effective == SYNC_ALL_COLUMNS; + if !sync && effective != APPEND_NEW_COLUMNS { + return Vec::new(); + } + findings + .iter() + .filter_map(|finding| { + let (kind, column_type) = match finding.kind { + SchemaChangeKind::ColumnAdded => { + (SchemaActionKind::AddColumn, finding.expected_type.clone()) + } + SchemaChangeKind::ColumnRemoved if sync => (SchemaActionKind::DropColumn, None), + SchemaChangeKind::ColumnTypeChanged if sync => ( + SchemaActionKind::AlterColumnType, + finding.expected_type.clone(), + ), + SchemaChangeKind::ColumnRemoved | SchemaChangeKind::ColumnTypeChanged => { + return None; + } + }; + Some(SchemaActionRow { + kind, + column_name: finding.column_name.clone(), + column_type, + }) + }) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/singular_attachment.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/singular_attachment.rs new file mode 100644 index 0000000000..cf6063ec4c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/singular_attachment.rs @@ -0,0 +1,63 @@ +//! Where a singular audit attaches, from its references and the model graph. + +use crate::planner_strategy::models::AuditAttachmentKind; +use crate::planner_strategy::types::Reaches; + +/// Whether every other referenced model reaches `candidate`. +fn reached_by_all( + candidate: &str, + models: &[String], + reaches: &Reaches<'_, E>, +) -> Result { + for other in models { + if other != candidate && !reaches(other, candidate)? { + return Ok(false); + } + } + Ok(true) +} + +/// The single model every other referenced model reaches, if there is one. +fn latest_owner(models: &[String], reaches: &Reaches<'_, E>) -> Result, E> { + let mut candidates: Vec<&String> = Vec::new(); + for candidate in models { + if reached_by_all(candidate, models, reaches)? { + candidates.push(candidate); + } + } + if candidates.len() == 1 { + return Ok(Some(candidates[0].clone())); + } + if candidates.is_empty() { + return Ok(None); + } + candidates.sort(); + let mut deepest: &String = candidates[0]; + for other in &candidates[1..] { + if reaches(deepest, other)? { + deepest = other; + } + } + Ok(reached_by_all(deepest, models, reaches)?.then(|| deepest.clone())) +} + +/// Python's `_infer_singular_attachment` over the audit's model and source reference names. +pub fn singular_attachment( + model_references: &[String], + source_references: &[String], + reaches: &Reaches<'_, E>, +) -> Result<(AuditAttachmentKind, Option), E> { + if model_references.is_empty() { + return Ok(match source_references.iter().min() { + Some(source) => (AuditAttachmentKind::Source, Some(source.clone())), + None => (AuditAttachmentKind::End, None), + }); + } + if let [model] = model_references { + return Ok((AuditAttachmentKind::Model, Some(model.clone()))); + } + Ok(match latest_owner(model_references, reaches)? { + Some(owner) => (AuditAttachmentKind::Model, Some(owner)), + None => (AuditAttachmentKind::End, None), + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/mod.rs b/crates/sqlbuild-analysis/src/planner_strategy/mod.rs new file mode 100644 index 0000000000..e7bd10bd20 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/mod.rs @@ -0,0 +1,11 @@ +//! Planner strategy: plan actions and reasons, schema actions, model warnings, audit attachment and run scope, cursor type checks, watermark lineage and microbatch counting and capping. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/models.rs b/crates/sqlbuild-analysis/src/planner_strategy/models.rs new file mode 100644 index 0000000000..5553b77357 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/models.rs @@ -0,0 +1,180 @@ +//! Planner strategy inputs and decisions. + +use crate::planner_changes::models::SchemaFindingRow; + +/// How a model is materialized; an unknown `materialized` value is a custom materialization. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MaterializationType { + View, + Table, + Incremental, + Snapshot, + Seed, + Custom, +} + +/// What the plan does with a model. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PlanAction { + CreateView, + CreateTable, + IncrementalAppend, + IncrementalDeleteInsert, + IncrementalMerge, + Snapshot, + Skip, + Custom, +} + +/// Why the plan does it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PlanReason { + FirstRun, + Renamed, + FullRefresh, + QueryChanged, + FunctionChanged, + ConfigChanged, + SchemaChanged, + NormalIncremental, + NoChange, + Disabled, +} + +/// How a model changed since its recorded build, including first runs and renames. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PlanChangeKind { + FirstRun, + Renamed, + QueryChanged, + FunctionChanged, + ConfigChanged, + SchemaChanged, + NoChange, +} + +/// The parts of a model's change result the strategy reads. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ChangeFacts { + pub kind: PlanChangeKind, + pub backfill_full: bool, + pub backfill_forward_only: bool, + pub query_changed: bool, + pub changed_functions: Vec, + pub config_changed: bool, + pub schema_findings: Vec, + pub recorded_build_relation_missing: bool, +} + +/// The model configuration the plan action depends on. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ModelStrategy { + pub name: String, + pub disabled: bool, + pub materialization: MaterializationType, + /// The `incremental_strategy` value when it is a string. + pub incremental_strategy: Option, +} + +/// What a schema action does to one column. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SchemaActionKind { + AddColumn, + DropColumn, + AlterColumnType, +} + +/// One schema change to apply to a target table. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SchemaActionRow { + pub kind: SchemaActionKind, + pub column_name: String, + pub column_type: Option, +} + +/// How severe a plan warning is. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WarningSeverity { + Error, + Warning, + Info, +} + +/// One plan warning about a model. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WarningRow { + pub severity: WarningSeverity, + pub message: String, + pub code: Option<&'static str>, +} + +/// The inputs of a model's plan warnings besides its change facts. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WarningInputs { + pub model_name: String, + pub materialization: MaterializationType, + pub has_schema_actions: bool, + pub on_schema_change: Option, + pub type_enforcement: bool, +} + +/// A cursor type. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CursorType { + Timestamp, + Integer, +} + +/// A model's cursor column and the warehouse columns its type is checked against. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CursorTypeCheck { + pub model_name: String, + pub cursor_column: Option, + pub cursor_type: Option, + pub warehouse_columns: Vec<(String, String)>, + pub sql_analysis_enabled: bool, +} + +/// Where an audit attaches. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AuditAttachmentKind { + Source, + Seed, + Model, + End, +} + +/// When an audit runs. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AuditRunScope { + Final, + DeltaAndFinal, +} + +/// The kind and name of one SQL reference. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct ReferenceIdentity { + pub kind: String, + pub name: String, +} + +/// The batch-limit facts of an upstream model. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProducerFacts { + pub microbatch: bool, + pub capped: bool, +} + +/// The end a microbatch cap keeps. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CapDirection { + FromStart, + FromEnd, +} + +/// The bound a cap moves: a new end when capping from the start, a new start when capping from the end. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CappedBound { + End(T), + Start(T), +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests.rs new file mode 100644 index 0000000000..30e1350d0c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_planner_strategy.rs"] +mod planner_strategy; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/helpers.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/helpers.rs new file mode 100644 index 0000000000..6d2c4b6c41 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/helpers.rs @@ -0,0 +1,115 @@ +use std::collections::BTreeMap; + +use crate::planner_changes::models::{SchemaChangeKind, SchemaColumnSource, SchemaFindingRow}; +use crate::planner_strategy::errors::LineageFailure; +use crate::planner_strategy::models::{ + ChangeFacts, PlanChangeKind, ProducerFacts, ReferenceIdentity, WarningRow, WarningSeverity, +}; +use crate::planner_strategy::tests::test_types::{Lineage, Steps}; +use crate::planner_strategy::types::{ReferenceGraph, Timeline}; + +pub(super) fn change(kind: PlanChangeKind) -> ChangeFacts { + ChangeFacts { + kind, + backfill_full: false, + backfill_forward_only: true, + query_changed: kind == PlanChangeKind::QueryChanged, + changed_functions: Vec::new(), + config_changed: false, + schema_findings: Vec::new(), + recorded_build_relation_missing: false, + } +} + +pub(super) fn finding( + kind: SchemaChangeKind, + source: SchemaColumnSource, + (expected_type, actual_type): (Option<&str>, Option<&str>), +) -> SchemaFindingRow { + SchemaFindingRow { + kind, + column_name: "amount".to_owned(), + source, + expected_type: expected_type.map(str::to_owned), + actual_type: actual_type.map(str::to_owned), + } +} + +pub(super) fn warning(severity: WarningSeverity, message: &str) -> WarningRow { + WarningRow { + severity, + message: message.to_owned(), + code: None, + } +} + +pub(super) fn names(items: &[&str]) -> Vec { + items.iter().map(|item| (*item).to_owned()).collect() +} + +impl Timeline for Steps { + type Moment = i64; + type Error = (); + + fn advance(&self, moment: &i64) -> Result { + Ok(moment + 3) + } + + fn precedes(&self, left: &i64, right: &i64) -> Result { + Ok(left < right) + } +} + +fn reference(kind: &str, name: &str) -> ReferenceIdentity { + ReferenceIdentity { + kind: kind.to_owned(), + name: name.to_owned(), + } +} + +impl Lineage { + pub(super) fn roots() -> Vec { + vec![reference("ref", "customers"), reference("udf", "discount")] + } +} + +impl ReferenceGraph for Lineage { + type Reference = ReferenceIdentity; + type Error = (); + + fn identity(&self, reference: &ReferenceIdentity) -> Result { + Ok(reference.clone()) + } + + fn model_references(&self, name: &str) -> Result>, ()> { + let models: BTreeMap<&str, Vec> = BTreeMap::from([ + ("customers", vec![reference("ref", "events")]), + ("events", vec![reference("source", "raw_events")]), + ("products", Vec::new()), + ]); + Ok(models.get(name).cloned()) + } + + fn function_references(&self, name: &str) -> Result>, ()> { + Ok((name == "discount").then(|| vec![reference("ref", "products")])) + } + + fn producer_facts(&self, name: &str) -> Result { + Ok(ProducerFacts { + microbatch: name == "customers" || self.capped.contains(name), + capped: self.capped.contains(name), + }) + } + + fn watermark_inputs(&self, name: &str) -> Result, ()> { + Ok(if name == "customers" { + names(&["events"]) + } else { + Vec::new() + }) + } +} + +pub(super) fn failure_text(failure: LineageFailure<()>) -> String { + format!("{failure:?}") +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs new file mode 100644 index 0000000000..a65170fcaf --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs @@ -0,0 +1,585 @@ +use std::collections::BTreeSet; + +use crate::planner_changes::models::{SchemaChangeKind, SchemaColumnSource}; +use crate::planner_strategy::constants::QUERY_CHANGE_TRACKING_HELP; +use crate::planner_strategy::main::cap_integer_bounds::cap_integer_bounds; +use crate::planner_strategy::main::cap_timeline_bounds::cap_timeline_bounds; +use crate::planner_strategy::main::capped_watermark_input::capped_watermark_input; +use crate::planner_strategy::main::classify_cursor_sql_type::classify_cursor_sql_type; +use crate::planner_strategy::main::count_integer_batches::count_integer_batches; +use crate::planner_strategy::main::count_timeline_batches::count_timeline_batches; +use crate::planner_strategy::main::lineage_reference::lineage_reference; +use crate::planner_strategy::main::microbatch_limit_warning::microbatch_limit_warning; +use crate::planner_strategy::main::model_plan_action::model_plan_action; +use crate::planner_strategy::main::model_warnings::model_warnings; +use crate::planner_strategy::main::schema_actions::schema_actions; +use crate::planner_strategy::main::singular_attachment::singular_attachment; +use crate::planner_strategy::models::{ + AuditAttachmentKind, CapDirection, CappedBound, ChangeFacts, CursorType, MaterializationType, + ModelStrategy, PlanAction, PlanChangeKind, PlanReason, SchemaActionKind, SchemaActionRow, + WarningInputs, WarningSeverity, +}; +use crate::planner_strategy::tests::helpers::{change, failure_text, finding, names, warning}; +use crate::planner_strategy::tests::test_types::{ + ActionTestCase, AttachmentTestCase, ClassifyTestCase, IntegerBatchTestCase, + LimitWarningTestCase, Lineage, LineageTestCase, SchemaActionTestCase, Steps, + TimelineBatchTestCase, WarningTestCase, +}; + +#[test] +fn given_model_change_when_resolving_action_then_matches_python_strategy() { + let test_cases = [ + ActionTestCase { + description: "view rename", + materialization: MaterializationType::View, + strategy: None, + change: change(PlanChangeKind::Renamed), + full_refresh: false, + expected_action: Ok((PlanAction::CreateView, PlanReason::Renamed)), + }, + ActionTestCase { + description: "snapshot without change", + materialization: MaterializationType::Snapshot, + strategy: None, + change: change(PlanChangeKind::NoChange), + full_refresh: false, + expected_action: Ok((PlanAction::Snapshot, PlanReason::NormalIncremental)), + }, + ActionTestCase { + description: "custom full refresh", + materialization: MaterializationType::Custom, + strategy: None, + change: change(PlanChangeKind::Renamed), + full_refresh: true, + expected_action: Ok((PlanAction::Custom, PlanReason::FullRefresh)), + }, + ActionTestCase { + description: "table query change", + materialization: MaterializationType::Table, + strategy: None, + change: change(PlanChangeKind::QueryChanged), + full_refresh: false, + expected_action: Ok((PlanAction::CreateTable, PlanReason::QueryChanged)), + }, + ActionTestCase { + description: "incremental full backfill of a config change", + materialization: MaterializationType::Incremental, + strategy: Some("merge"), + change: ChangeFacts { + backfill_full: true, + config_changed: true, + ..change(PlanChangeKind::ConfigChanged) + }, + full_refresh: false, + expected_action: Ok((PlanAction::CreateTable, PlanReason::ConfigChanged)), + }, + ActionTestCase { + description: "incremental append", + materialization: MaterializationType::Incremental, + strategy: Some("append"), + change: change(PlanChangeKind::NoChange), + full_refresh: false, + expected_action: Ok((PlanAction::IncrementalAppend, PlanReason::NormalIncremental)), + }, + ActionTestCase { + description: "incremental without strategy", + materialization: MaterializationType::Incremental, + strategy: None, + change: change(PlanChangeKind::NoChange), + full_refresh: false, + expected_action: Err( + "S201 incremental model 'orders' is missing required incremental_strategy" + .to_owned(), + ), + }, + ActionTestCase { + description: "incremental with unknown strategy", + materialization: MaterializationType::Incremental, + strategy: Some("upsert"), + change: change(PlanChangeKind::QueryChanged), + full_refresh: false, + expected_action: Err( + "S202 incremental model 'orders' has unknown strategy 'upsert'".to_owned(), + ), + }, + ]; + for test_case in test_cases { + let model = ModelStrategy { + name: "orders".to_owned(), + disabled: false, + materialization: test_case.materialization, + incremental_strategy: test_case.strategy.map(str::to_owned), + }; + assert_eq!( + model_plan_action(&model, &test_case.change, test_case.full_refresh) + .map_err(|error| format!("{} {}", error.code(), error.message())), + test_case.expected_action, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_schema_findings_when_resolving_actions_then_follows_on_schema_change() { + let added = SchemaActionRow { + kind: SchemaActionKind::AddColumn, + column_name: "amount".to_owned(), + column_type: Some("INTEGER".to_owned()), + }; + let test_cases = [ + SchemaActionTestCase { + description: "default appends new columns", + on_schema_change: None, + expected_actions: vec![added.clone()], + }, + SchemaActionTestCase { + description: "sync drops and retypes", + on_schema_change: Some("sync_all_columns"), + expected_actions: vec![ + added, + SchemaActionRow { + kind: SchemaActionKind::DropColumn, + column_name: "amount".to_owned(), + column_type: None, + }, + SchemaActionRow { + kind: SchemaActionKind::AlterColumnType, + column_name: "amount".to_owned(), + column_type: Some("BIGINT".to_owned()), + }, + ], + }, + SchemaActionTestCase { + description: "fail resolves nothing", + on_schema_change: Some("fail"), + expected_actions: Vec::new(), + }, + ]; + let findings = [ + finding( + SchemaChangeKind::ColumnAdded, + SchemaColumnSource::Yml, + (Some("INTEGER"), None), + ), + finding( + SchemaChangeKind::ColumnRemoved, + SchemaColumnSource::Yml, + (None, Some("INTEGER")), + ), + finding( + SchemaChangeKind::ColumnTypeChanged, + SchemaColumnSource::Yml, + (Some("BIGINT"), Some("INTEGER")), + ), + ]; + for test_case in test_cases { + assert_eq!( + schema_actions(&findings, test_case.on_schema_change), + test_case.expected_actions, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_change_facts_when_building_warnings_then_matches_python_messages() { + let test_cases = [ + WarningTestCase { + description: "ignored inferred addition", + on_schema_change: Some("ignore"), + type_enforcement: false, + change: ChangeFacts { + schema_findings: vec![finding( + SchemaChangeKind::ColumnAdded, + SchemaColumnSource::SqlAnalysis, + (Some(""), None), + )], + ..change(PlanChangeKind::SchemaChanged) + }, + expected_warnings: vec![ + warning( + WarningSeverity::Info, + "inferred column amount: new column not in warehouse", + ), + warning( + WarningSeverity::Info, + "schema change detected but on_schema_change is set to ignore", + ), + ], + }, + WarningTestCase { + description: "failing enforced type change", + on_schema_change: Some("fail"), + type_enforcement: true, + change: ChangeFacts { + schema_findings: vec![finding( + SchemaChangeKind::ColumnTypeChanged, + SchemaColumnSource::Yml, + (Some("BIGINT"), None), + )], + ..change(PlanChangeKind::SchemaChanged) + }, + expected_warnings: vec![ + warning( + WarningSeverity::Error, + "schema change detected and on_schema_change is set to fail", + ), + warning( + WarningSeverity::Warning, + "enforced column amount type mismatch: expected BIGINT, warehouse has None", + ), + ], + }, + WarningTestCase { + description: "forward-only function change", + on_schema_change: None, + type_enforcement: false, + change: ChangeFacts { + changed_functions: names(&["discount", "tax"]), + ..change(PlanChangeKind::FunctionChanged) + }, + expected_warnings: vec![warning( + WarningSeverity::Warning, + &format!( + "orders: function discount, tax changed, so already-built dates keep their \ + old results - only new dates will use the new query. To apply the change \ + to existing data, choose how much to reprocess: replay_on_change full \ + (rebuild every date with the new SQL), replay_on_change bounded- \ + e.g. bounded-14d (reprocess just that window), or leave it forward-only \ + (existing dates as-is, new dates use the new query - the default). If this \ + model's output does not depend on the SQL that changed, this is safe to \ + ignore. {QUERY_CHANGE_TRACKING_HELP}" + ), + )], + }, + ]; + for test_case in test_cases { + let inputs = WarningInputs { + model_name: "orders".to_owned(), + materialization: MaterializationType::Incremental, + has_schema_actions: false, + on_schema_change: test_case.on_schema_change.map(str::to_owned), + type_enforcement: test_case.type_enforcement, + }; + assert_eq!( + model_warnings(&inputs, &test_case.change), + test_case.expected_warnings, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_warehouse_type_when_classifying_cursor_type_then_matches_python() { + let test_cases = [ + ClassifyTestCase { + description: "timestamp with time zone", + sql_type: "timestamp with time zone", + expected_cursor_type: Some(CursorType::Timestamp), + }, + ClassifyTestCase { + description: "number(38, 0)", + sql_type: "number(38, 0)", + expected_cursor_type: Some(CursorType::Integer), + }, + ClassifyTestCase { + description: "NUMERIC(10,2)", + sql_type: "NUMERIC(10,2)", + expected_cursor_type: None, + }, + ClassifyTestCase { + description: "decimal", + sql_type: "decimal", + expected_cursor_type: Some(CursorType::Integer), + }, + ClassifyTestCase { + description: " big int ", + sql_type: " big int ", + expected_cursor_type: Some(CursorType::Integer), + }, + ClassifyTestCase { + description: "VARCHAR(20)", + sql_type: "VARCHAR(20)", + expected_cursor_type: None, + }, + ClassifyTestCase { + description: "INTEGER", + sql_type: "INTEGER", + expected_cursor_type: Some(CursorType::Integer), + }, + ]; + for test_case in test_cases { + assert_eq!( + classify_cursor_sql_type(test_case.sql_type).ok().flatten(), + test_case.expected_cursor_type, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_integer_range_when_counting_and_capping_then_matches_python() { + let test_cases = [ + IntegerBatchTestCase { + description: "partial last batch from the end", + bounds: (0, 10), + batch_size: " 3 ", + direction: CapDirection::FromEnd, + expected_count: 4, + expected_cap: Some(CappedBound::Start(6)), + }, + IntegerBatchTestCase { + description: "from the start", + bounds: (0, 10), + batch_size: "1_0", + direction: CapDirection::FromStart, + expected_count: 1, + expected_cap: Some(CappedBound::End(10)), + }, + IntegerBatchTestCase { + description: "unparsable size", + bounds: (0, 10), + batch_size: "1d", + direction: CapDirection::FromStart, + expected_count: 0, + expected_cap: None, + }, + IntegerBatchTestCase { + description: "zero size", + bounds: (0, 10), + batch_size: "0", + direction: CapDirection::FromEnd, + expected_count: 0, + expected_cap: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + count_integer_batches(test_case.bounds, test_case.batch_size, false), + test_case.expected_count, + "{}", + test_case.description + ); + assert_eq!( + cap_integer_bounds( + test_case.bounds, + test_case.batch_size, + 2, + test_case.direction + ), + test_case.expected_cap, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_timeline_range_when_counting_and_capping_then_clips_the_last_batch() { + let test_cases = [ + TimelineBatchTestCase { + description: "last two of four batches", + bounds: (0, 10), + max_batches: 2, + direction: CapDirection::FromEnd, + expected_count: 4, + expected_cap: CappedBound::Start(6), + }, + TimelineBatchTestCase { + description: "first two batches", + bounds: (0, 10), + max_batches: 2, + direction: CapDirection::FromStart, + expected_count: 4, + expected_cap: CappedBound::End(6), + }, + TimelineBatchTestCase { + description: "more allowed than planned", + bounds: (0, 4), + max_batches: 5, + direction: CapDirection::FromStart, + expected_count: 2, + expected_cap: CappedBound::End(4), + }, + TimelineBatchTestCase { + description: "equal bounds", + bounds: (5, 5), + max_batches: 1, + direction: CapDirection::FromEnd, + expected_count: 1, + expected_cap: CappedBound::Start(5), + }, + ]; + for test_case in test_cases { + assert_eq!( + count_timeline_batches(&Steps, test_case.bounds, true), + Ok(test_case.expected_count), + "{}", + test_case.description + ); + assert_eq!( + cap_timeline_bounds( + &Steps, + test_case.bounds, + test_case.max_batches, + test_case.direction + ), + Ok(test_case.expected_cap), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_microbatch_limit_when_exceeded_then_formats_python_warning() { + let test_cases = [ + LimitWarningTestCase { + description: "within the limit", + limit: (Some(2), 2), + action: ("warn", false), + expected_warning: None, + }, + LimitWarningTestCase { + description: "no limit", + limit: (None, 9), + action: ("error", false), + expected_warning: None, + }, + LimitWarningTestCase { + description: "capped", + limit: (Some(2), 5), + action: ("cap_from_end", true), + expected_warning: Some( + "MICROBATCH LIMIT EXCEEDED: model 'orders' planned 5 batches, above the \ + per-model limit of 2 (action=cap_from_end). Selected 2 batches and deferred 3.", + ), + }, + ]; + for test_case in test_cases { + assert_eq!( + microbatch_limit_warning("orders", test_case.limit, test_case.action).as_deref(), + test_case.expected_warning, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_audit_references_when_inferring_attachment_then_picks_latest_owner() { + let test_cases = [ + AttachmentTestCase { + description: "sources only", + models: Vec::new(), + sources: names(&["payments", "events"]), + expected_attachment: (AuditAttachmentKind::Source, Some("events".to_owned())), + }, + AttachmentTestCase { + description: "nothing", + models: Vec::new(), + sources: Vec::new(), + expected_attachment: (AuditAttachmentKind::End, None), + }, + AttachmentTestCase { + description: "downstream model owns", + models: names(&["orders", "customers"]), + sources: Vec::new(), + expected_attachment: (AuditAttachmentKind::Model, Some("orders".to_owned())), + }, + AttachmentTestCase { + description: "duplicate reference", + models: names(&["products", "products"]), + sources: Vec::new(), + expected_attachment: (AuditAttachmentKind::Model, Some("products".to_owned())), + }, + AttachmentTestCase { + description: "unrelated models", + models: names(&["products", "customers"]), + sources: Vec::new(), + expected_attachment: (AuditAttachmentKind::End, None), + }, + ]; + for test_case in test_cases { + let reaches = |from: &str, to: &str| -> Result { + Ok(from == "customers" && to == "orders") + }; + assert_eq!( + singular_attachment(&test_case.models, &test_case.sources, &reaches), + Ok(test_case.expected_attachment), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_watermark_input_when_walking_lineage_then_finds_reference_and_capped_producer() { + let test_cases = [ + LineageTestCase { + description: "through a function", + input: "products", + capped: &[], + expected_reference: Ok("products".to_owned()), + expected_capped: Ok(None), + }, + LineageTestCase { + description: "capped direct producer", + input: "customers", + capped: &["customers"], + expected_reference: Ok("customers".to_owned()), + expected_capped: Ok(Some("customers".to_owned())), + }, + LineageTestCase { + description: "capped producer behind a microbatch watermark", + input: "customers", + capped: &["events"], + expected_reference: Ok("customers".to_owned()), + expected_capped: Ok(Some("events".to_owned())), + }, + LineageTestCase { + description: "missing input", + input: "returns", + capped: &[], + expected_reference: Err( + "NotInLineage { model: \"orders\", input: \"returns\" }".to_owned() + ), + expected_capped: Err( + "NotInLineage { model: \"orders\", input: \"returns\" }".to_owned() + ), + }, + ]; + for test_case in test_cases { + let graph = Lineage { + capped: test_case + .capped + .iter() + .map(|name| (*name).to_owned()) + .collect::>(), + }; + assert_eq!( + lineage_reference(&graph, ("orders", Lineage::roots()), test_case.input) + .map(|reference| reference.name) + .map_err(failure_text), + test_case.expected_reference, + "{}", + test_case.description + ); + assert_eq!( + capped_watermark_input( + &graph, + ("orders", Lineage::roots()), + names(&[test_case.input]) + ) + .map(|found| found.map(|capped| capped.producer)) + .map_err(failure_text), + test_case.expected_capped, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs new file mode 100644 index 0000000000..e28e6bc03a --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs @@ -0,0 +1,83 @@ +use std::collections::BTreeSet; + +use crate::planner_strategy::models::{ + AuditAttachmentKind, CapDirection, CappedBound, ChangeFacts, CursorType, MaterializationType, + PlanAction, PlanReason, SchemaActionRow, WarningRow, +}; + +pub(super) struct ActionTestCase { + pub(super) description: &'static str, + pub(super) materialization: MaterializationType, + pub(super) strategy: Option<&'static str>, + pub(super) change: ChangeFacts, + pub(super) full_refresh: bool, + pub(super) expected_action: Result<(PlanAction, PlanReason), String>, +} + +pub(super) struct SchemaActionTestCase { + pub(super) description: &'static str, + pub(super) on_schema_change: Option<&'static str>, + pub(super) expected_actions: Vec, +} + +pub(super) struct WarningTestCase { + pub(super) description: &'static str, + pub(super) on_schema_change: Option<&'static str>, + pub(super) type_enforcement: bool, + pub(super) change: ChangeFacts, + pub(super) expected_warnings: Vec, +} + +pub(super) struct ClassifyTestCase { + pub(super) description: &'static str, + pub(super) sql_type: &'static str, + pub(super) expected_cursor_type: Option, +} + +pub(super) struct IntegerBatchTestCase { + pub(super) description: &'static str, + pub(super) bounds: (i128, i128), + pub(super) batch_size: &'static str, + pub(super) direction: CapDirection, + pub(super) expected_count: i128, + pub(super) expected_cap: Option>, +} + +pub(super) struct TimelineBatchTestCase { + pub(super) description: &'static str, + pub(super) bounds: (i64, i64), + pub(super) max_batches: i128, + pub(super) direction: CapDirection, + pub(super) expected_count: i128, + pub(super) expected_cap: CappedBound, +} + +pub(super) struct AttachmentTestCase { + pub(super) description: &'static str, + pub(super) models: Vec, + pub(super) sources: Vec, + pub(super) expected_attachment: (AuditAttachmentKind, Option), +} + +pub(super) struct LineageTestCase { + pub(super) description: &'static str, + pub(super) input: &'static str, + pub(super) capped: &'static [&'static str], + pub(super) expected_reference: Result, + pub(super) expected_capped: Result, String>, +} + +pub(super) struct LimitWarningTestCase { + pub(super) description: &'static str, + pub(super) limit: (Option, i128), + pub(super) action: (&'static str, bool), + pub(super) expected_warning: Option<&'static str>, +} + +/// Whole numbers stepping by three. +pub(super) struct Steps; + +/// `orders` reads `customers` and the `discount` function; `discount` reads `products`; `customers` is a microbatch model reading `events` as its watermark input. +pub(super) struct Lineage { + pub(super) capped: BTreeSet, +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/types.rs b/crates/sqlbuild-analysis/src/planner_strategy/types.rs new file mode 100644 index 0000000000..a8f5860b0d --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/types.rs @@ -0,0 +1,39 @@ +//! Planner strategy callback types. + +use crate::planner_strategy::models::{ProducerFacts, ReferenceIdentity}; + +/// Whether the second model name is downstream of the first. +pub type Reaches<'a, E> = dyn Fn(&str, &str) -> Result + 'a; + +/// A timeline of cursor moments the caller owns: one batch step and the ordering. +pub trait Timeline { + type Moment: Clone; + type Error; + + /// The moment one batch after `moment`. + fn advance(&self, moment: &Self::Moment) -> Result; + + /// Whether `left` sorts before `right`. + fn precedes(&self, left: &Self::Moment, right: &Self::Moment) -> Result; +} + +/// The project's SQL references as the caller holds them. +pub trait ReferenceGraph { + type Reference: Clone; + type Error; + + /// The kind and name of a reference. + fn identity(&self, reference: &Self::Reference) -> Result; + + /// The references of a model, when the project has it. + fn model_references(&self, name: &str) -> Result>, Self::Error>; + + /// The references of a SQL function, when the project has it. + fn function_references(&self, name: &str) -> Result>, Self::Error>; + + /// Whether a model is a microbatch model and caps its batches. + fn producer_facts(&self, name: &str) -> Result; + + /// The watermark input names of a model. + fn watermark_inputs(&self, name: &str) -> Result, Self::Error>; +} diff --git a/crates/sqlbuild-analysis/src/python_plan/main/mod.rs b/crates/sqlbuild-analysis/src/python_plan/main/mod.rs index 8cb536efba..3c8238dc13 100644 --- a/crates/sqlbuild-analysis/src/python_plan/main/mod.rs +++ b/crates/sqlbuild-analysis/src/python_plan/main/mod.rs @@ -1,2 +1,3 @@ pub mod ordered_python_names; +pub mod run_lifecycle; pub mod upstream_closure; diff --git a/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs b/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs new file mode 100644 index 0000000000..0f33cbb1ab --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs @@ -0,0 +1,46 @@ +//! Classify selected run nodes into lifecycle-aware execution regions. + +use std::collections::{BTreeSet, HashMap}; + +use crate::python_plan::main::upstream_closure::upstream_closure; +use crate::python_plan::models::{LifecycleNode, RunLifecycle}; +use crate::python_selection::constants::{ASSET_KIND, LOADER_KIND, TASK_KIND}; + +/// Ingress is every selected warehouse loader and its selected upstream tasks, assets and loaders. +#[must_use] +pub fn run_lifecycle( + selected: &BTreeSet, + nodes: &HashMap, + upstream: &HashMap>, +) -> RunLifecycle { + let kind = |name: &str| nodes.get(name).map_or("", |node| node.kind.as_str()); + let loaders: BTreeSet = selected + .iter() + .filter(|name| { + nodes + .get(name.as_str()) + .is_some_and(|node| node.kind == LOADER_KIND && !node.external_loader) + }) + .cloned() + .collect(); + let mut ingress: BTreeSet = loaders.clone(); + for loader in &loaders { + for name in upstream_closure(loader, upstream) { + if selected.contains(&name) + && [TASK_KIND, ASSET_KIND, LOADER_KIND].contains(&kind(&name)) + { + ingress.insert(name); + } + } + } + let read_side: Vec = selected + .iter() + .filter(|name| !ingress.contains(*name) && [TASK_KIND, ASSET_KIND].contains(&kind(name))) + .cloned() + .collect(); + RunLifecycle { + ingress_names: ingress.into_iter().collect(), + ingress_loader_names: loaders.into_iter().collect(), + read_side_names: read_side, + } +} diff --git a/crates/sqlbuild-analysis/src/python_plan/mod.rs b/crates/sqlbuild-analysis/src/python_plan/mod.rs index e29b572755..1a70a48981 100644 --- a/crates/sqlbuild-analysis/src/python_plan/mod.rs +++ b/crates/sqlbuild-analysis/src/python_plan/mod.rs @@ -1,4 +1,5 @@ pub mod main; +pub mod models; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-analysis/src/python_plan/models.rs b/crates/sqlbuild-analysis/src/python_plan/models.rs new file mode 100644 index 0000000000..21b5825300 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/models.rs @@ -0,0 +1,18 @@ +//! Run lifecycle inputs and the regions selected Python nodes run in. + +/// One Python node as lifecycle classification reads it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LifecycleNode { + /// `task`, `asset`, `loader` or `check`. + pub kind: String, + /// Whether the node is a loader that owns its own warehouse connection. + pub external_loader: bool, +} + +/// Selected Python nodes split into ingress and read-side regions, each sorted. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RunLifecycle { + pub ingress_names: Vec, + pub ingress_loader_names: Vec, + pub read_side_names: Vec, +} diff --git a/crates/sqlbuild-analysis/src/python_plan/tests.rs b/crates/sqlbuild-analysis/src/python_plan/tests.rs index 4f8ba4548a..fc29fd608e 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests.rs @@ -2,5 +2,7 @@ mod helpers; #[path = "tests/test_python_plan.rs"] mod python_plan; +#[path = "tests/test_run_lifecycle.rs"] +mod run_lifecycle; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs b/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs index 55a49940b2..4489f0f16c 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs @@ -1,5 +1,7 @@ use std::collections::HashMap; +use crate::python_plan::models::LifecycleNode; + pub(crate) fn edges(rows: &[(&str, &[&str])]) -> HashMap> { rows.iter() .map(|(name, upstream)| { @@ -10,3 +12,17 @@ pub(crate) fn edges(rows: &[(&str, &[&str])]) -> HashMap> { }) .collect() } + +pub(crate) fn lifecycle_nodes(rows: &[(&str, &str, bool)]) -> HashMap { + rows.iter() + .map(|(name, kind, external_loader)| { + ( + (*name).to_owned(), + LifecycleNode { + kind: (*kind).to_owned(), + external_loader: *external_loader, + }, + ) + }) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs b/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs new file mode 100644 index 0000000000..c79fd71b91 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs @@ -0,0 +1,87 @@ +use std::collections::BTreeSet; + +use crate::python_plan::main::run_lifecycle::run_lifecycle; +use crate::python_plan::tests::helpers::{edges, lifecycle_nodes}; +use crate::python_plan::tests::test_types::LifecycleTestCase; + +#[test] +fn given_selection_when_classifying_lifecycle_then_matches_python_regions() { + let test_cases = [ + LifecycleTestCase { + description: "a warehouse loader pulls its selected upstream tasks into ingress", + selected: &["fetch_rates", "load_orders", "report", "check_orders"], + nodes: &[ + ("fetch_rates", "task", false), + ("load_orders", "loader", false), + ("report", "asset", false), + ("check_orders", "check", false), + ], + upstream: &[("load_orders", &["fetch_rates", "unselected"])], + expected_ingress: &["fetch_rates", "load_orders"], + expected_loaders: &["load_orders"], + expected_read_side: &["report"], + }, + LifecycleTestCase { + description: "external loaders are neither ingress loaders nor read-side nodes", + selected: &["sync_orders", "export"], + nodes: &[("sync_orders", "loader", true), ("export", "task", false)], + upstream: &[("export", &["sync_orders"])], + expected_ingress: &[], + expected_loaders: &[], + expected_read_side: &["export"], + }, + LifecycleTestCase { + description: "a selected external loader upstream of a warehouse loader is ingress", + selected: &["sync_orders", "load_orders"], + nodes: &[ + ("sync_orders", "loader", true), + ("load_orders", "loader", false), + ], + upstream: &[("load_orders", &["sync_orders"])], + expected_ingress: &["load_orders", "sync_orders"], + expected_loaders: &["load_orders"], + expected_read_side: &[], + }, + ]; + + for test_case in test_cases { + let selected: BTreeSet = test_case + .selected + .iter() + .map(|name| (*name).to_owned()) + .collect(); + + let lifecycle = run_lifecycle( + &selected, + &lifecycle_nodes(test_case.nodes), + &edges(test_case.upstream), + ); + + assert_eq!( + ( + lifecycle.ingress_names, + lifecycle.ingress_loader_names, + lifecycle.read_side_names, + ), + ( + test_case + .expected_ingress + .iter() + .map(|name| (*name).to_owned()) + .collect::>(), + test_case + .expected_loaders + .iter() + .map(|name| (*name).to_owned()) + .collect::>(), + test_case + .expected_read_side + .iter() + .map(|name| (*name).to_owned()) + .collect::>(), + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs b/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs index 7cab9c803f..7a4dd8f19a 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs @@ -11,3 +11,14 @@ pub(crate) struct ClosureTestCase { pub(crate) upstream: &'static [(&'static str, &'static [&'static str])], pub(crate) expected_closure: &'static [&'static str], } + +pub(crate) struct LifecycleTestCase { + pub(crate) description: &'static str, + pub(crate) selected: &'static [&'static str], + /// `(name, kind, external loader)`. + pub(crate) nodes: &'static [(&'static str, &'static str, bool)], + pub(crate) upstream: &'static [(&'static str, &'static [&'static str])], + pub(crate) expected_ingress: &'static [&'static str], + pub(crate) expected_loaders: &'static [&'static str], + pub(crate) expected_read_side: &'static [&'static str], +} diff --git a/crates/sqlbuild-analysis/src/type_system/main/mod.rs b/crates/sqlbuild-analysis/src/type_system/main/mod.rs index 81be6727b9..85cc669961 100644 --- a/crates/sqlbuild-analysis/src/type_system/main/mod.rs +++ b/crates/sqlbuild-analysis/src/type_system/main/mod.rs @@ -1 +1,2 @@ pub mod normalize_type; +pub mod polyglot_type_args; diff --git a/crates/sqlbuild-analysis/src/type_system/main/polyglot_type_args.rs b/crates/sqlbuild-analysis/src/type_system/main/polyglot_type_args.rs new file mode 100644 index 0000000000..972ac1f6c0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/main/polyglot_type_args.rs @@ -0,0 +1,17 @@ +//! The scalar arguments of a type as Polyglot parses it. + +use polyglot_sql::{DataType, Dialect, DialectType}; +use serde_json::{Map, Value}; + +use crate::type_system::_helpers::parsed::parse_type; +use crate::type_system::_helpers::shallow_args::shallow_args; + +/// `parse_data_type(type_sql, dialect=...).args` scalars, or the parse error text. +pub fn polyglot_type_args(type_sql: &str, dialect: &str) -> Result, String> { + let polyglot: Dialect = dialect + .parse::() + .map(Dialect::get) + .map_err(|_| format!("unknown dialect '{dialect}'"))?; + let parsed: DataType = parse_type(type_sql, &polyglot)?; + Ok(shallow_args(&parsed)) +} diff --git a/crates/sqlbuild-cli/src/argv/_helpers/formatting.rs b/crates/sqlbuild-cli/src/argv/_helpers/formatting.rs index a8a487eb9a..fc5e1b4606 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/formatting.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/formatting.rs @@ -136,6 +136,7 @@ fn format_text(formatter: &Formatter, text: &str, indent: i64) -> String { &normalize_whitespace(text), width as usize, (&padding, &padding), + true, ); format!("{}\n\n", lines.join("\n")) } @@ -185,7 +186,14 @@ fn format_action(formatter: &Formatter, entry: &HelpEntry, indent: i64) -> Strin } }; let lines: Vec = help - .map(|help| text_wrap(&normalize_whitespace(help), help_width as usize, ("", ""))) + .map(|help| { + text_wrap( + &normalize_whitespace(help), + help_width as usize, + ("", ""), + true, + ) + }) .unwrap_or_default(); if let Some((first, rest)) = lines.split_first() { text.push_str(&format!("{}{first}\n", " ".repeat(indent_first as usize))); diff --git a/crates/sqlbuild-cli/src/argv/tests/test_text_wrap.rs b/crates/sqlbuild-cli/src/argv/tests/test_text_wrap.rs index ce6cb6fe10..53eac1b414 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_text_wrap.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_text_wrap.rs @@ -34,7 +34,8 @@ fn given_help_text_when_wrapping_then_lines_match_python_textwrap() { text_wrap( &normalize_whitespace(test_case.text), test_case.width, - (test_case.indent, test_case.indent) + (test_case.indent, test_case.indent), + true ), test_case.expected_lines, "{}", diff --git a/crates/sqlbuild-core/src/build_progress/_helpers/audits.rs b/crates/sqlbuild-core/src/build_progress/_helpers/audits.rs new file mode 100644 index 0000000000..aeb8401a10 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/_helpers/audits.rs @@ -0,0 +1,92 @@ +//! Per-batch delta audit results folded into one display row per audit and phase. + +use crate::build_progress::constants::{DELTA_AND_FINAL, ERROR, INSUFFICIENT, PASS, WARN}; +use crate::build_progress::models::AuditResultText; + +/// One aggregated audit row. +pub(crate) struct AuditDisplayEntry { + pub(crate) label: &'static str, + pub(crate) display_name: String, + pub(crate) outcome: &'static str, + pub(crate) total_row_count: i64, + pub(crate) batch_pass: usize, + pub(crate) batch_total: usize, + pub(crate) reused: bool, + pub(crate) executed_sql: Option, +} + +/// Audits sharing a name, column and run-scope phase. +type AuditGroup<'a> = ( + (&'a str, Option<&'a str>, &'a str), + Vec<&'a AuditResultText>, +); + +fn worst_outcome(results: &[&AuditResultText]) -> &'static str { + for outcome in [ERROR, WARN, INSUFFICIENT] { + if results.iter().any(|result| result.outcome == outcome) { + return outcome; + } + } + PASS +} + +fn phase_label(has_delta: bool, phase: &str) -> &'static str { + match (has_delta, phase == DELTA_AND_FINAL) { + (false, _) => "audit", + (true, true) => "audit (d)", + (true, false) => "audit (f)", + } +} + +fn display_entry( + (name, column, phase): (&str, Option<&str>, &str), + results: &[&AuditResultText], + has_delta: bool, +) -> AuditDisplayEntry { + let mut total_row_count: i64 = 0; + let mut batch_pass: usize = 0; + let mut reused: bool = true; + for result in results { + total_row_count += result.row_count; + batch_pass += usize::from(result.outcome == PASS); + reused = reused && result.reused; + } + AuditDisplayEntry { + label: phase_label(has_delta, phase), + display_name: match column { + Some(column) => format!("{name} ({column})"), + None => name.to_owned(), + }, + outcome: worst_outcome(results), + total_row_count, + batch_pass, + batch_total: results.len(), + reused, + executed_sql: results + .first() + .and_then(|result| result.executed_sql.clone()), + } +} + +/// Group audits by name, column and run-scope phase in first-seen order. +pub(crate) fn aggregate_audit_results(audits: &[AuditResultText]) -> Vec { + let has_delta: bool = audits + .iter() + .any(|audit| audit.run_scope_phase == DELTA_AND_FINAL); + let mut groups: Vec> = Vec::new(); + for audit in audits { + let key = ( + audit.audit_name.as_str(), + audit.attached_column_name.as_deref(), + audit.run_scope_phase.as_str(), + ); + match groups.iter_mut().find(|(existing, _)| *existing == key) { + Some((_, members)) => members.push(audit), + None => groups.push((key, vec![audit])), + } + } + groups + .into_iter() + .map(|(key, results)| display_entry(key, &results, has_delta)) + .collect() +} diff --git a/crates/sqlbuild-core/src/build_progress/_helpers/error_rows.rs b/crates/sqlbuild-core/src/build_progress/_helpers/error_rows.rs new file mode 100644 index 0000000000..0f354fa692 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/_helpers/error_rows.rs @@ -0,0 +1,83 @@ +//! Inline error rows under progress lines and error blocks of the failure summary. + +use crate::build_progress::constants::TYPE_WIDTH; +use crate::build_progress::models::ErrorText; +use crate::cli_text::main::format_coded_error::format_coded_error; +use crate::cli_text::main::format_inline_error_lines::format_inline_error_lines; +use crate::cli_text::main::format_tree_connector::format_tree_connector; +use crate::cli_text::models::CliStyle; +use crate::text::main::python_splitlines::python_splitlines; + +/// The `error` label and wrapped message under a row, optionally drawn as a tree child. +pub(crate) fn error_detail( + (error, message): (&ErrorText, &str), + (prefix_width, columns): (usize, usize), + tree_child: bool, + style: CliStyle, +) -> String { + let pad: String = " ".repeat(prefix_width + if tree_child { 4 } else { 0 }); + let connector_suffix: String = if tree_child { + format!("{} ", format_tree_connector(style, true)) + } else { + String::new() + }; + let label_width: usize = if tree_child { + TYPE_WIDTH - 3 + } else { + TYPE_WIDTH - 1 + }; + let label: String = format!( + "{}{}", + style.error_muted("error"), + " ".repeat(label_width.saturating_sub("error".len())) + ); + let plain_prefix_width: usize = pad.len() + if tree_child { 4 } else { 0 } + label_width + 1; + let content_width: usize = (columns as i64 - plain_prefix_width as i64).max(1) as usize; + let lines: Vec = format_inline_error_lines( + (error.code.as_deref(), message, error.help.as_deref()), + content_width, + style, + ); + let continuation: String = " ".repeat(plain_prefix_width); + let mut text: String = String::new(); + for (index, line) in lines.iter().enumerate() { + if index == 0 { + text.push_str(&format!("{pad}{connector_suffix}{label} {line}\n")); + } else { + text.push_str(&format!("{continuation}{line}\n")); + } + } + text +} + +/// The labelled, indented error block of one failure summary entry. +pub(crate) fn failure_error_block( + (code, message, help): (Option<&str>, &str, Option<&str>), + style: CliStyle, +) -> Vec { + let label: String = format!( + "{}{}", + style.error_muted("error"), + " ".repeat(TYPE_WIDTH - "error".len()) + ); + let rendered: String = match code { + None => message.to_owned(), + Some(code) => format_coded_error((code, message, help), false, style), + }; + let mut lines: Vec<&str> = python_splitlines(&rendered); + if lines.is_empty() { + lines.push(&rendered); + } + lines + .iter() + .enumerate() + .map(|(index, line)| { + let display_label: String = if index == 0 { + label.clone() + } else { + " ".repeat(TYPE_WIDTH) + }; + format!(" {display_label}{line}") + }) + .collect() +} diff --git a/crates/sqlbuild-core/src/build_progress/_helpers/footer.rs b/crates/sqlbuild-core/src/build_progress/_helpers/footer.rs new file mode 100644 index 0000000000..f3601560cb --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/_helpers/footer.rs @@ -0,0 +1,344 @@ +//! Counts, warnings, failures and the skipped rollup of the build footer. + +use crate::build_progress::_helpers::error_rows::failure_error_block; +use crate::build_progress::_helpers::text::pad; +use crate::build_progress::constants::{ + DELTA_SUFFIX, ERROR, FAILED, PASS, PYTHON_NODE_NAME_WIDTH, SKIPPED, SKIPPED_ROLLUP_CAP, + SUCCESS, WARN, +}; +use crate::build_progress::models::{AuditResultText, BuildResultText, ErrorText, PythonNodeText}; +use crate::cli_text::models::CliStyle; + +/// Pass, warn, fail and skip counts of the footer. +#[derive(Default)] +pub(crate) struct FooterCounts { + pub(crate) pass: i64, + pub(crate) warn: i64, + pub(crate) fail: i64, + pub(crate) skip: i64, +} + +impl FooterCounts { + fn status(&mut self, status: &str, warned: bool) { + match status { + SUCCESS if warned => self.warn += 1, + SUCCESS => self.pass += 1, + FAILED => self.fail += 1, + SKIPPED => self.skip += 1, + _ => {} + } + } +} + +fn build_audits(result: &BuildResultText) -> impl Iterator { + result + .models + .iter() + .flat_map(|model| model.audits.iter()) + .chain(result.source_audits.iter()) + .chain(result.end_audits.iter()) +} + +pub(crate) fn count_results(result: &BuildResultText, python: &[PythonNodeText]) -> FooterCounts { + let mut counts: FooterCounts = FooterCounts::default(); + for model in &result.models { + counts.status(&model.status, !model.warnings.is_empty()); + } + for audit in build_audits(result) { + match audit.outcome.as_str() { + PASS => counts.pass += 1, + WARN => counts.warn += 1, + ERROR => counts.fail += 1, + _ => {} + } + } + for seed in &result.seeds { + counts.status(&seed.status, false); + } + for function in &result.functions { + counts.status(&function.status, false); + counts.warn += function.warnings.len() as i64; + } + for load in &result.loads { + counts.status(&load.status, !load.warnings.is_empty()); + } + for test in &result.tests { + if test.outcome == PASS { + counts.pass += 1; + } else { + counts.fail += 1; + } + } + for node in python { + counts.status(&node.status, !node.warnings.is_empty()); + } + counts +} + +/// One titled footer section that is shown only when it has entries. +struct Section { + title: String, + lines: Vec, +} + +impl Section { + fn new(title: String) -> Self { + Self { + title, + lines: Vec::new(), + } + } + + fn entry(&mut self, heading: String, body: impl IntoIterator) { + self.lines.push(heading); + self.lines.extend(body); + self.lines.push(String::new()); + } + + fn failure(&mut self, heading: String, error: &ErrorText, style: CliStyle) { + self.failure_with(heading, error, None, style); + } + + fn failure_with( + &mut self, + heading: String, + error: &ErrorText, + extra: Option, + style: CliStyle, + ) { + let mut body: Vec = Vec::new(); + if let Some(message) = &error.message { + body.extend(failure_error_block( + (error.code.as_deref(), message, error.help.as_deref()), + style, + )); + } + body.extend(extra); + self.entry(heading, body); + } + + fn finish(self) -> Vec { + if self.lines.is_empty() { + return Vec::new(); + } + let mut lines: Vec = vec![String::new(), self.title, String::new()]; + lines.extend(self.lines); + lines + } +} + +fn audit_warning(audit: &AuditResultText) -> String { + let name: String = match audit.attached_column_name.as_deref() { + Some(column) if !column.is_empty() => format!("{} ({column})", audit.audit_name), + _ => audit.audit_name.clone(), + }; + if audit.measurement { + return format!(" audit {name} measured {}", audit.measured_value); + } + let row_label: &str = if audit.row_count == 1 { "row" } else { "rows" }; + format!(" audit {name} returned {} {row_label}", audit.row_count) +} + +pub(crate) fn warning_details( + result: &BuildResultText, + python: &[PythonNodeText], + style: CliStyle, +) -> Vec { + let mut section: Section = Section::new(style.warning_strong("Warnings:")); + for model in &result.models { + let mut warnings: Vec = model + .audits + .iter() + .filter(|audit| audit.outcome == WARN) + .map(audit_warning) + .collect(); + warnings.extend( + model + .warnings + .iter() + .map(|warning| format!(" {warning}")), + ); + if !warnings.is_empty() { + section.entry(format!(" {}", model.name), warnings); + } + } + let labelled = result + .functions + .iter() + .map(|function| { + ( + format!("{} ({})", function.name, function.kind), + &function.warnings, + ) + }) + .chain(result.loads.iter().map(|load| { + ( + format!("{} (loader {})", load.source_name, load.loader_name), + &load.warnings, + ) + })) + .chain( + python + .iter() + .map(|node| (format!("{} ({})", node.name, node.kind), &node.warnings)), + ); + for (label, messages) in labelled { + if !messages.is_empty() { + section.entry( + format!(" {label}"), + messages.iter().map(|message| format!(" {message}")), + ); + } + } + section.finish() +} + +fn inspection_relation_message(relation: &str) -> String { + if relation.ends_with(DELTA_SUFFIX) { + return format!("delta table kept for inspection: {relation}"); + } + format!("staging table kept for inspection: {relation}") +} + +fn message_error(message: Option<&String>) -> ErrorText { + ErrorText { + message: message.cloned(), + ..ErrorText::default() + } +} + +pub(crate) fn failure_details(result: &BuildResultText, style: CliStyle) -> Vec { + let mut section: Section = Section::new(style.error_strong("Failures:")); + for load in result.loads.iter().filter(|load| load.status == FAILED) { + section.failure( + format!(" {} ({})", load.source_name, load.resource_kind), + &message_error(load.error_message.as_ref()), + style, + ); + } + for seed in result.seeds.iter().filter(|seed| seed.status == FAILED) { + section.failure(format!(" {} (seed)", seed.name), &seed.error, style); + } + for model in result.models.iter().filter(|model| model.status == FAILED) { + let phase: String = match model.failed_phase.as_deref() { + Some(phase) if !phase.is_empty() => format!(" ({phase})"), + _ => String::new(), + }; + section.failure_with( + format!(" {}{phase}", model.name), + &model.error, + model + .staging_relation + .as_deref() + .map(|relation| format!(" {}", inspection_relation_message(relation))), + style, + ); + } + for function in result + .functions + .iter() + .filter(|function| function.status == FAILED) + { + section.failure( + format!(" {} ({})", function.name, function.kind), + &function.error, + style, + ); + } + for test in result.tests.iter().filter(|test| test.outcome != PASS) { + section.failure(format!(" {} (test)", test.label), &test.error, style); + } + for audit in result.source_audits.iter().chain(result.end_audits.iter()) { + if audit.execution_error.is_none() { + continue; + } + let target: String = audit + .attached_target_name + .as_deref() + .map_or_else(String::new, |target| format!(" on {target}")); + section.failure( + format!(" {}{target} (audit)", audit.audit_name), + &message_error(audit.execution_error.as_ref()), + style, + ); + } + section.finish() +} + +pub(crate) fn python_failure_details(python: &[PythonNodeText], style: CliStyle) -> Vec { + let failed: Vec<&PythonNodeText> = python.iter().filter(|node| node.status == FAILED).collect(); + if failed.is_empty() { + return Vec::new(); + } + let mut lines: Vec = vec![ + String::new(), + style.error_strong("Python node failures:"), + String::new(), + ]; + for node in failed { + let message: &str = match node.error_message.as_deref() { + Some(message) if !message.is_empty() => message, + _ => "Python node failed", + }; + lines.push(format!( + " {}{} {}", + pad(&node.kind, 10), + pad(&node.name, PYTHON_NODE_NAME_WIDTH), + style.error(message) + )); + } + lines +} + +pub(crate) fn skipped_rollup( + result: &BuildResultText, + python: &[PythonNodeText], + style: CliStyle, +) -> Vec { + let names: Vec<&str> = result + .models + .iter() + .filter(|model| model.status == SKIPPED) + .map(|model| model.name.as_str()) + .chain( + result + .seeds + .iter() + .filter(|seed| seed.status == SKIPPED) + .map(|seed| seed.name.as_str()), + ) + .chain( + result + .functions + .iter() + .filter(|function| function.status == SKIPPED) + .map(|function| function.name.as_str()), + ) + .chain( + result + .loads + .iter() + .filter(|load| load.status == SKIPPED) + .map(|load| load.source_name.as_str()), + ) + .chain( + python + .iter() + .filter(|node| node.status == SKIPPED) + .map(|node| node.name.as_str()), + ) + .collect(); + if names.is_empty() { + return Vec::new(); + } + let visible: Vec<&str> = names.iter().take(SKIPPED_ROLLUP_CAP).copied().collect(); + let remaining: usize = names.len() - visible.len(); + let mut rendered: String = visible.join(", "); + if remaining > 0 { + rendered = format!("{rendered}, ... (+{remaining} more)"); + } + vec![ + String::new(), + style.muted(&format!("Skipped ({}): {rendered}", names.len())), + ] +} diff --git a/crates/sqlbuild-core/src/build_progress/_helpers/mod.rs b/crates/sqlbuild-core/src/build_progress/_helpers/mod.rs new file mode 100644 index 0000000000..fe214acff5 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod audits; +pub(crate) mod error_rows; +pub(crate) mod footer; +pub(crate) mod model_rows; +pub(crate) mod rows; +pub(crate) mod text; diff --git a/crates/sqlbuild-core/src/build_progress/_helpers/model_rows.rs b/crates/sqlbuild-core/src/build_progress/_helpers/model_rows.rs new file mode 100644 index 0000000000..152aa2def0 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/_helpers/model_rows.rs @@ -0,0 +1,248 @@ +//! The rows nested under a finished model row. + +use crate::build_progress::_helpers::audits::{AuditDisplayEntry, aggregate_audit_results}; +use crate::build_progress::_helpers::rows::{TopLevelRow, log_block, sql_block, top_level_line}; +use crate::build_progress::_helpers::text::{ + abbreviated_rows, audit_outcome_display, counter_text, execution_status_display, + format_duration, pad, skip_detail, test_outcome_display, truncate_name, +}; +use crate::build_progress::constants::{ + BATCH_INDENT, FAILED, HOOK_MAX_LABEL_WIDTH, HOOK_MIN_LABEL_WIDTH, HOOK_PHASE_WIDTH, + HOOK_TYPE_WIDTH, PASS, SKIPPED, TYPE_WIDTH, +}; +use crate::build_progress::models::{ + HookResultText, LifecycleEventText, ModelResultText, ModelRowFacts, ProgressLayout, + TestResultText, +}; +use crate::cli_text::models::CliStyle; +use crate::plan_display::main::format_no_input_rows::format_no_input_rows; +use crate::plan_display::main::format_waiting_on_empty_inputs::format_waiting_on_empty_inputs; +use crate::progress::main::format_expectation_name::format_expectation_name; + +/// The indent and name width of rows nested under the model row. +pub(crate) struct SubRows { + pub(crate) pad: String, + pub(crate) name_width: usize, + pub(crate) style: CliStyle, +} + +pub(crate) fn streaming_batch_summary(model: &ModelResultText) -> Option { + let batch_count: i64 = model.batch_count?; + let label: &str = if batch_count == 1 { "batch" } else { "batches" }; + let mut count_text: String = format!("{batch_count} {label}"); + if let Some(size) = &model.batch_size { + count_text = format!("{count_text} ({size})"); + } + let mut parts: Vec = vec![count_text]; + if let Some((start, end)) = &model.cursor_range + && model.empty_cursor_inputs.is_empty() + { + parts.push(format!("range {start} \u{2192} {end}")); + } + if let Some(rows) = model.rows_affected { + parts.push(abbreviated_rows(rows)); + } + if !model.empty_cursor_inputs.is_empty() && !model.waiting_on_empty_inputs { + parts.push(format_no_input_rows(&model.empty_cursor_inputs)); + } + if model.recovery_batch_count != 0 { + parts.push(format!("{} recovery", model.recovery_batch_count)); + } + if model.synthetic_completion_count != 0 { + parts.push(format!( + "{} synthetic coverage", + model.synthetic_completion_count + )); + } + if let Some(state) = &model.replay_requirement_state { + parts.push(format!("replay {state}")); + } + Some(format!("{BATCH_INDENT}{}", parts.join(" "))) +} + +pub(crate) fn empty_cursor_inputs(model: &ModelResultText) -> Option { + if model.empty_cursor_inputs.is_empty() || model.waiting_on_empty_inputs { + return None; + } + Some(format!( + "{BATCH_INDENT}{}", + format_no_input_rows(&model.empty_cursor_inputs) + )) +} + +fn hook_label_width(hooks: &[HookResultText]) -> usize { + let Some(longest) = hooks.iter().map(|hook| hook.label.chars().count()).max() else { + return HOOK_MIN_LABEL_WIDTH; + }; + HOOK_MIN_LABEL_WIDTH.max(longest.min(HOOK_MAX_LABEL_WIDTH)) +} + +pub(crate) fn hook_rows(hooks: &[HookResultText], pre_hook: bool, sub: &SubRows) -> String { + let width: usize = hook_label_width(hooks); + let label: &str = if pre_hook { "pre_hook" } else { "post_hook" }; + let mut text: String = String::new(); + for hook in hooks.iter().filter(|hook| hook.pre_hook == pre_hook) { + let detail: String = if hook.status == SKIPPED { + skip_detail(&hook.skip) + } else { + String::new() + }; + text.push_str(&format!( + "{}{}{}{} {}{detail}\n", + sub.pad, + sub.style.muted(&pad(label, HOOK_PHASE_WIDTH)), + sub.style.muted(&pad(&hook.hook_type, HOOK_TYPE_WIDTH)), + pad(&truncate_name(&hook.label, width), width), + sub.style.status(&execution_status_display(&hook.status)) + )); + } + text +} + +pub(crate) fn test_rows(tests: &[TestResultText], sub: &SubRows) -> String { + let style: CliStyle = sub.style; + let mut text: String = String::new(); + for test in tests { + text.push_str(&format!( + "{}{}{} {}\n", + sub.pad, + style.muted(&pad("test", TYPE_WIDTH)), + pad(&test.label, sub.name_width), + style.status(test_outcome_display(&test.outcome)) + )); + for step in &test.steps { + text.push_str(&format!( + "{} {}{} {}{}\n", + sub.pad, + style.muted(&pad("expect", TYPE_WIDTH - 2)), + pad(&format_expectation_name(&step.model_name), sub.name_width), + style.status(test_outcome_display(&step.outcome)), + step.detail + )); + } + } + text +} + +pub(crate) fn audit_row(entry: &AuditDisplayEntry, sub: &SubRows) -> String { + let mut detail: String = String::new(); + if entry.reused { + detail = " proof reused".to_owned(); + } + if entry.outcome != PASS && entry.total_row_count > 0 { + let label: &str = if entry.total_row_count == 1 { + "row" + } else { + "rows" + }; + detail = format!(" {} {label}", entry.total_row_count); + } + if entry.batch_total > 1 { + detail = format!(" {}/{}{detail}", entry.batch_pass, entry.batch_total); + } + format!( + "{}{}{} {}{detail}\n", + sub.pad, + sub.style.muted(&pad(entry.label, TYPE_WIDTH)), + pad( + &truncate_name(&entry.display_name, sub.name_width), + sub.name_width + ), + sub.style.status(&audit_outcome_display(entry.outcome)) + ) +} + +/// The model row itself, with its annotation, phase or skip detail. +pub(crate) fn model_line( + (model, facts): (&ModelResultText, &ModelRowFacts), + counter: usize, + layout: ProgressLayout, + style: CliStyle, +) -> String { + let mut name: String = model.name.clone(); + if !facts.annotation.is_empty() { + name = format!("{name} ({})", facts.annotation); + } + let mut duration: String = format_duration(model.duration_ms); + let mut detail: String = String::new(); + if model.status == FAILED + && let Some(phase) = &model.failed_phase + { + detail = format!(" {phase}"); + } else if model.status == SKIPPED { + duration = String::new(); + detail = skip_detail(&model.skip); + } + top_level_line( + &TopLevelRow { + counter: &counter_text(counter, layout.total), + resource_type: &facts.display_type, + name: &truncate_name(&name, layout.name_width), + status: &execution_status_display(&model.status), + duration: &duration, + detail: &detail, + }, + layout, + style, + ) +} + +/// Lifecycle SQL and log blocks, falling back to the planned DDL. +pub(crate) fn verbose_blocks( + model: &ModelResultText, + facts: &ModelRowFacts, + style: CliStyle, +) -> String { + let fallback: Vec = facts + .logical_ddl + .iter() + .map(|ddl| LifecycleEventText::Sql(ddl.clone())) + .collect(); + let events: &[LifecycleEventText] = if model.events.is_empty() { + &fallback + } else { + &model.events + }; + events + .iter() + .map(|event| match event { + LifecycleEventText::Sql(sql) => sql_block(sql, style), + LifecycleEventText::Log(message) => log_block(message, style), + }) + .collect() +} + +/// The batch summary or empty-input note, and the waiting-on-empty-inputs warning. +pub(crate) fn batch_lines(model: &ModelResultText, style: CliStyle) -> String { + let mut text: String = String::new(); + if let Some(line) = streaming_batch_summary(model).or_else(|| empty_cursor_inputs(model)) { + text.push_str(&format!("{}\n", style.muted(&line))); + } + if model.waiting_on_empty_inputs && !model.empty_cursor_inputs.is_empty() { + let line: String = format!( + "{BATCH_INDENT}{}", + format_waiting_on_empty_inputs(&model.empty_cursor_inputs) + ); + text.push_str(&format!("{}\n", style.warning(&line))); + } + text +} + +/// Pre-hooks, tests, audits with their verbose SQL, then post-hooks. +pub(crate) fn nested_rows( + model: &ModelResultText, + tests: &[TestResultText], + sub: &SubRows, + verbose: bool, +) -> String { + let mut text: String = hook_rows(&model.hooks, true, sub); + text.push_str(&test_rows(tests, sub)); + for entry in aggregate_audit_results(&model.audits) { + text.push_str(&audit_row(&entry, sub)); + if verbose && let Some(sql) = &entry.executed_sql { + text.push_str(&sql_block(sql, sub.style)); + } + } + text.push_str(&hook_rows(&model.hooks, false, sub)); + text +} diff --git a/crates/sqlbuild-core/src/build_progress/_helpers/rows.rs b/crates/sqlbuild-core/src/build_progress/_helpers/rows.rs new file mode 100644 index 0000000000..8f33a80252 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/_helpers/rows.rs @@ -0,0 +1,82 @@ +//! Row pieces shared by the persistent build progress lines. + +use crate::build_progress::_helpers::text::{counter_width, display_sql, pad}; +use crate::build_progress::constants::{LOG_CONTINUATION, LOG_LABEL, TYPE_WIDTH}; +use crate::build_progress::models::{LifecycleEventText, ProgressLayout}; +use crate::cli_text::main::format_status_cell::format_status_cell; +use crate::cli_text::models::CliStyle; +use crate::text::main::python_splitlines::python_splitlines; + +/// The width of the ` N/M ` prefix every top-level row starts with. +pub(crate) fn prefix_width(layout: ProgressLayout) -> usize { + 2 + counter_width(layout.total) + 2 +} + +/// The cells of one top-level result row. +pub(crate) struct TopLevelRow<'a> { + pub(crate) counter: &'a str, + pub(crate) resource_type: &'a str, + pub(crate) name: &'a str, + pub(crate) status: &'a str, + pub(crate) duration: &'a str, + pub(crate) detail: &'a str, +} + +pub(crate) fn top_level_line( + row: &TopLevelRow<'_>, + layout: ProgressLayout, + style: CliStyle, +) -> String { + let duration: String = if row.duration.is_empty() { + String::new() + } else { + style.muted(row.duration) + }; + format!( + " {} {}{} {} {duration}{}\n", + row.counter, + style.muted(&pad(row.resource_type, TYPE_WIDTH)), + pad(row.name, layout.name_width), + format_status_cell(style, row.status, 6), + row.detail + ) +} + +pub(crate) fn sql_block(sql: &str, style: CliStyle) -> String { + let mut text: String = "\n".to_owned(); + for line in display_sql(sql).split('\n') { + text.push_str(&format!("{}\n", style.muted(&format!(" {line}")))); + } + text.push('\n'); + text +} + +pub(crate) fn log_block(message: &str, style: CliStyle) -> String { + let lines: Vec<&str> = python_splitlines(message); + let first: &str = lines.first().copied().unwrap_or(""); + let mut text: String = format!("\n{}{}\n", style.label(LOG_LABEL), style.muted(first)); + for line in lines.iter().skip(1) { + text.push_str(&format!( + "{}\n", + style.muted(&format!("{LOG_CONTINUATION}{line}")) + )); + } + text +} + +/// Log blocks, and SQL blocks when verbose, of a lifecycle event list. +pub(crate) fn event_blocks( + events: &[LifecycleEventText], + verbose: bool, + style: CliStyle, +) -> String { + let mut text: String = String::new(); + for event in events { + match event { + LifecycleEventText::Log(message) => text.push_str(&log_block(message, style)), + LifecycleEventText::Sql(sql) if verbose => text.push_str(&sql_block(sql, style)), + LifecycleEventText::Sql(_) => {} + } + } + text +} diff --git a/crates/sqlbuild-core/src/build_progress/_helpers/text.rs b/crates/sqlbuild-core/src/build_progress/_helpers/text.rs new file mode 100644 index 0000000000..ce726f2a01 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/_helpers/text.rs @@ -0,0 +1,142 @@ +//! Python formatting idioms and small labels shared by build progress rows. + +use crate::build_progress::constants::{ + ERROR, FAILED, INSUFFICIENT, PASS, ROW_COUNT_ABBREVIATION_THRESHOLD, ROW_COUNT_BILLION, + ROW_COUNT_MILLION, ROW_COUNT_THOUSAND, SECONDS_PER_MINUTE, SKIPPED, SUCCESS, WARN, +}; +use crate::build_progress::models::SkipText; +use crate::text::main::python_isspace::python_isspace; + +/// `f"{text: String { + let length: usize = text.chars().count(); + format!("{text}{}", " ".repeat(width.saturating_sub(length))) +} + +/// `f"{text:>width}"`: right-aligned, padded by character count. +pub(crate) fn pad_left(text: &str, width: usize) -> String { + let length: usize = text.chars().count(); + format!("{}{text}", " ".repeat(width.saturating_sub(length))) +} + +/// `text[:count]` by character. +pub(crate) fn head(text: &str, count: usize) -> String { + text.chars().take(count).collect() +} + +/// `f"{value:,}"`: the integer with comma-separated thousands. +pub(crate) fn group_thousands(value: i64) -> String { + let digits: String = value.unsigned_abs().to_string(); + let mut grouped: String = String::new(); + for (index, digit) in digits.chars().enumerate() { + if index > 0 && (digits.len() - index).is_multiple_of(3) { + grouped.push(','); + } + grouped.push(digit); + } + if value < 0 { + format!("-{grouped}") + } else { + grouped + } +} + +/// The name cut to `width` characters with a trailing ellipsis. +pub(crate) fn truncate_name(name: &str, width: usize) -> String { + if name.chars().count() <= width { + return name.to_owned(); + } + format!("{}...", head(name, width.saturating_sub(3))) +} + +/// `"{counter}/{total}"` right-aligned to twice the total's digits plus one. +pub(crate) fn counter_text(counter: usize, total: usize) -> String { + pad_left(&format!("{counter}/{total}"), counter_width(total)) +} + +pub(crate) fn counter_width(total: usize) -> usize { + total.to_string().len() * 2 + 1 +} + +/// Seconds with two decimals under a minute, minutes and seconds otherwise. +pub(crate) fn format_duration(duration_ms: Option) -> String { + let Some(duration_ms) = duration_ms else { + return String::new(); + }; + let seconds: f64 = duration_ms / 1000.0; + if seconds < SECONDS_PER_MINUTE { + return format!("{seconds:.2}s"); + } + let minutes: i64 = (seconds / SECONDS_PER_MINUTE).floor() as i64; + let remaining: f64 = seconds - minutes as f64 * SECONDS_PER_MINUTE; + format!("{minutes}m{remaining:.1}s") +} + +/// The OK, FAIL or SKIP word of an execution status. +pub(crate) fn execution_status_display(status: &str) -> String { + match status { + SUCCESS => "OK".to_owned(), + FAILED => "FAIL".to_owned(), + SKIPPED => "SKIP".to_owned(), + other => other.to_owned(), + } +} + +/// The status word of an audit outcome. +pub(crate) fn audit_outcome_display(outcome: &str) -> String { + match outcome { + PASS => "PASS".to_owned(), + WARN => "WARN".to_owned(), + ERROR => "FAIL".to_owned(), + INSUFFICIENT => "INSUFFICIENT".to_owned(), + other => other.to_owned(), + } +} + +/// The status word of a SQL test outcome. +pub(crate) fn test_outcome_display(outcome: &str) -> &'static str { + match outcome { + PASS => "PASS", + ERROR => "ERROR", + _ => "FAIL", + } +} + +/// ` skip: ` for a skipped row. +pub(crate) fn skip_detail(skip: &SkipText) -> String { + let label: String = skip + .mode + .as_ref() + .map_or_else(|| "skip".to_owned(), |mode| format!("{mode} skip")); + match skip.reason.as_deref() { + Some(reason) if !reason.is_empty() => format!(" {label}: {reason}"), + _ => format!(" {label}"), + } +} + +/// The SQL with trailing whitespace removed and a terminating semicolon. +pub(crate) fn display_sql(sql: &str) -> String { + let stripped: &str = sql.trim_end_matches(python_isspace); + if stripped.is_empty() { + return sql.to_owned(); + } + if stripped.ends_with(';') { + return stripped.to_owned(); + } + format!("{stripped};") +} + +/// A row count with K/M/B abbreviation from ten thousand rows. +pub(crate) fn abbreviated_rows(count: i64) -> String { + let label: &str = if count == 1 { "row" } else { "rows" }; + if count >= ROW_COUNT_BILLION { + return format!("{:.1}B {label}", count as f64 / ROW_COUNT_BILLION as f64); + } + if count >= ROW_COUNT_MILLION { + return format!("{:.1}M {label}", count as f64 / ROW_COUNT_MILLION as f64); + } + if count >= ROW_COUNT_ABBREVIATION_THRESHOLD { + return format!("{:.1}K {label}", count as f64 / ROW_COUNT_THOUSAND as f64); + } + format!("{} {label}", group_thousands(count)) +} diff --git a/crates/sqlbuild-core/src/build_progress/constants.rs b/crates/sqlbuild-core/src/build_progress/constants.rs new file mode 100644 index 0000000000..648027a09d --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/constants.rs @@ -0,0 +1,37 @@ +//! Column widths, limits and status values of build progress output. + +pub(crate) const TYPE_WIDTH: usize = 10; +pub(crate) const MIN_NAME_WIDTH: usize = 20; +pub(crate) const NAME_PADDING: usize = 2; +pub(crate) const SUB_INDENT: usize = 2; +pub(crate) const TREE_CHILD_NAME_INDENT: usize = SUB_INDENT + 4; +pub(crate) const HOOK_PHASE_WIDTH: usize = 10; +pub(crate) const HOOK_TYPE_WIDTH: usize = 8; +pub(crate) const HOOK_MIN_LABEL_WIDTH: usize = 24; +pub(crate) const HOOK_MAX_LABEL_WIDTH: usize = 56; +pub(crate) const SKIPPED_ROLLUP_CAP: usize = 20; +pub(crate) const ROW_COUNT_BILLION: i64 = 1_000_000_000; +pub(crate) const ROW_COUNT_MILLION: i64 = 1_000_000; +pub(crate) const ROW_COUNT_THOUSAND: i64 = 1_000; +pub(crate) const ROW_COUNT_ABBREVIATION_THRESHOLD: i64 = 10_000; +pub(crate) const SECONDS_PER_MINUTE: f64 = 60.0; +pub(crate) const SCHEDULER_DIAGNOSTIC_LIMIT: usize = 50; +pub(crate) const QUERY_DIAGNOSTIC_LIMIT: usize = 25; +pub(crate) const SPINNER_FRAMES: [&str; 10] = [ + "\u{280b}", "\u{2819}", "\u{2839}", "\u{2838}", "\u{283c}", "\u{2834}", "\u{2826}", "\u{2827}", + "\u{2807}", "\u{280f}", +]; +pub(crate) const BATCH_INDENT: &str = " "; +pub(crate) const LOG_LABEL: &str = " log "; +pub(crate) const LOG_CONTINUATION: &str = " "; +pub(crate) const PYTHON_NODE_NAME_WIDTH: usize = 50; +pub(crate) const DELTA_SUFFIX: &str = "__delta"; + +pub(crate) const SUCCESS: &str = "success"; +pub(crate) const FAILED: &str = "failed"; +pub(crate) const SKIPPED: &str = "skipped"; +pub(crate) const PASS: &str = "pass"; +pub(crate) const WARN: &str = "warn"; +pub(crate) const ERROR: &str = "error"; +pub(crate) const INSUFFICIENT: &str = "insufficient"; +pub(crate) const DELTA_AND_FINAL: &str = "delta_and_final"; diff --git a/crates/sqlbuild-core/src/build_progress/main/format_build_footer.rs b/crates/sqlbuild-core/src/build_progress/main/format_build_footer.rs new file mode 100644 index 0000000000..97255700f9 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/format_build_footer.rs @@ -0,0 +1,57 @@ +//! The completion line, warnings, failures and skipped rollup printed after a build. + +use crate::build_progress::_helpers::footer::{ + FooterCounts, count_results, failure_details, python_failure_details, skipped_rollup, + warning_details, +}; +use crate::build_progress::constants::FAILED; +use crate::build_progress::models::{BuildResultText, PythonNodeText}; +use crate::cli_text::main::format_completion_line::format_completion_line; +use crate::cli_text::main::format_summary_footer::format_summary_footer; +use crate::cli_text::models::{CliStyle, CompletionState, SummaryCount}; + +fn count(label: &str, value: i64) -> SummaryCount { + SummaryCount { + label: label.to_owned(), + value, + } +} + +/// The build footer for `result` and the Python node results run alongside it. +#[must_use] +pub fn format_build_footer( + result: &BuildResultText, + python: &[PythonNodeText], + elapsed: f64, + style: CliStyle, +) -> String { + let counts: FooterCounts = count_results(result, python); + let insufficient: i64 = result.insufficient_count; + let total: i64 = counts.pass + counts.warn + counts.fail + counts.skip; + let summary: String = format_summary_footer( + style, + &[ + count("PASS", counts.pass), + count("WARN", counts.warn), + count("FAIL", counts.fail), + count("INSUFFICIENT", insufficient), + count("SKIP", counts.skip), + count("TOTAL", total + insufficient), + ], + Some(&format!("{elapsed:.2}s")), + ); + let python_failed: bool = python.iter().any(|node| node.status == FAILED); + let (state, label) = if result.failed || python_failed { + (CompletionState::Fail, "Completed with errors") + } else if result.warning_count > 0 || counts.warn > 0 { + (CompletionState::Warn, "Completed with warnings") + } else { + (CompletionState::Ok, "Completed successfully") + }; + let mut lines: Vec = warning_details(result, python, style); + lines.push(format_completion_line(style, state, label, Some(&summary))); + lines.extend(failure_details(result, style)); + lines.extend(python_failure_details(python, style)); + lines.extend(skipped_rollup(result, python, style)); + lines.join("\n") +} diff --git a/crates/sqlbuild-core/src/build_progress/main/mod.rs b/crates/sqlbuild-core/src/build_progress/main/mod.rs new file mode 100644 index 0000000000..09e6110e07 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/mod.rs @@ -0,0 +1,9 @@ +pub mod format_build_footer; +pub mod progress_name_width; +pub mod query_diagnostic_message; +pub mod render_build_spinner_line; +pub mod render_failed_test_result; +pub mod render_model_result; +pub mod render_resource_result; +pub mod runtime_diagnostic_summary; +pub mod scheduler_diagnostic_message; diff --git a/crates/sqlbuild-core/src/build_progress/main/progress_name_width.rs b/crates/sqlbuild-core/src/build_progress/main/progress_name_width.rs new file mode 100644 index 0000000000..973ec3533b --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/progress_name_width.rs @@ -0,0 +1,16 @@ +//! The shared name column width of build progress rows. + +use crate::build_progress::constants::{MIN_NAME_WIDTH, NAME_PADDING, TREE_CHILD_NAME_INDENT}; +use crate::build_progress::models::NameWidthInputs; +use crate::progress::main::format_expectation_name::format_expectation_name; + +/// The longest row name plus padding, so tree children still align with top-level statuses. +#[must_use] +pub fn progress_name_width(inputs: &NameWidthInputs) -> usize { + let top_level = inputs.names.iter().map(|name| name.chars().count()); + let children = inputs.tree_child_names.iter().map(|name| { + format_expectation_name(name).chars().count() + TREE_CHILD_NAME_INDENT - NAME_PADDING + }); + let longest: usize = top_level.chain(children).max().unwrap_or(0); + (longest + NAME_PADDING).max(MIN_NAME_WIDTH) +} diff --git a/crates/sqlbuild-core/src/build_progress/main/query_diagnostic_message.rs b/crates/sqlbuild-core/src/build_progress/main/query_diagnostic_message.rs new file mode 100644 index 0000000000..31259bccfc --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/query_diagnostic_message.rs @@ -0,0 +1,20 @@ +//! The diagnostic line of one completed warehouse statement. + +use crate::build_progress::models::StatementText; + +/// The resource, phase, query identity, status and elapsed time of a statement. +#[must_use] +pub fn query_diagnostic_message(statement: &StatementText) -> String { + let query_id: &str = match statement.query_id.as_deref() { + Some(query_id) if !query_id.is_empty() => query_id, + _ => "query ID unavailable", + }; + format!( + "Query {} {} phase={} {query_id} {} {:.2}s", + statement.resource_type, + statement.resource_name, + statement.phase, + statement.status, + statement.elapsed_seconds + ) +} diff --git a/crates/sqlbuild-core/src/build_progress/main/render_build_spinner_line.rs b/crates/sqlbuild-core/src/build_progress/main/render_build_spinner_line.rs new file mode 100644 index 0000000000..7a1852f4a5 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/render_build_spinner_line.rs @@ -0,0 +1,32 @@ +//! The live spinner row of the running build node. + +use crate::build_progress::_helpers::text::{counter_text, head, pad, truncate_name}; +use crate::build_progress::constants::{MIN_NAME_WIDTH, SPINNER_FRAMES, TYPE_WIDTH}; +use crate::build_progress::models::{ProgressLayout, SpinnerRow}; +use crate::cli_text::models::CliStyle; + +/// The cleared line with the position, kind, name, message and spinner frame `frame`. +#[must_use] +pub fn render_build_spinner_line( + row: &SpinnerRow, + frame: usize, + layout: ProgressLayout, + style: CliStyle, +) -> String { + let mut name: String = truncate_name(&row.name, layout.name_width); + if !row.message.is_empty() { + name = format!("{name} {}", row.message); + } + let mut plain: String = format!( + " {} {}{} ", + counter_text(row.position, layout.total), + pad(&row.resource_kind, TYPE_WIDTH), + pad(&name, layout.name_width) + ); + let max_width: i64 = (layout.columns as i64 - 2).max(MIN_NAME_WIDTH as i64); + if plain.chars().count() as i64 > max_width { + plain = format!("{}... ", head(&plain, (max_width - 4).max(0) as usize)); + } + let spinner: &str = SPINNER_FRAMES[frame % SPINNER_FRAMES.len()]; + format!("\r\x1b[K{plain}{}", style.status(spinner)) +} diff --git a/crates/sqlbuild-core/src/build_progress/main/render_failed_test_result.rs b/crates/sqlbuild-core/src/build_progress/main/render_failed_test_result.rs new file mode 100644 index 0000000000..375d60934b --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/render_failed_test_result.rs @@ -0,0 +1,54 @@ +//! A failing SQL test row written inline, with its expectations as tree children. + +use crate::build_progress::_helpers::error_rows::error_detail; +use crate::build_progress::_helpers::rows::prefix_width; +use crate::build_progress::_helpers::text::{ + counter_width, pad, test_outcome_display, truncate_name, +}; +use crate::build_progress::constants::{SUB_INDENT, TREE_CHILD_NAME_INDENT, TYPE_WIDTH}; +use crate::build_progress::models::{ProgressLayout, TestResultText}; +use crate::cli_text::main::format_tree_connector::format_tree_connector; +use crate::cli_text::models::CliStyle; +use crate::progress::main::format_expectation_name::format_expectation_name; + +/// The test row without a position, its expectation rows and its error. +#[must_use] +pub fn render_failed_test_result( + test: &TestResultText, + layout: ProgressLayout, + style: CliStyle, +) -> String { + let mut text: String = format!( + " {} {}{} {}\n", + " ".repeat(counter_width(layout.total)), + style.muted(&pad("test", TYPE_WIDTH)), + pad( + &truncate_name(&test.label, layout.name_width), + layout.name_width + ), + style.status(test_outcome_display(&test.outcome)) + ); + let error: Option<&str> = test.error.message.as_deref().filter(|m| !m.is_empty()); + let expectation_pad: String = " ".repeat(prefix_width(layout) + SUB_INDENT + 2); + let child_width: usize = layout.name_width.saturating_sub(TREE_CHILD_NAME_INDENT); + for (index, step) in test.steps.iter().enumerate() { + let last: bool = index + 1 == test.steps.len() && error.is_none(); + text.push_str(&format!( + "{expectation_pad}{} {}{} {}{}\n", + format_tree_connector(style, last), + style.muted(&pad("expect", TYPE_WIDTH - 2)), + pad(&format_expectation_name(&step.model_name), child_width), + style.status(test_outcome_display(&step.outcome)), + step.detail + )); + } + if let Some(message) = error { + text.push_str(&error_detail( + (&test.error, message), + (prefix_width(layout), layout.columns), + true, + style, + )); + } + text +} diff --git a/crates/sqlbuild-core/src/build_progress/main/render_model_result.rs b/crates/sqlbuild-core/src/build_progress/main/render_model_result.rs new file mode 100644 index 0000000000..62cb034bce --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/render_model_result.rs @@ -0,0 +1,44 @@ +//! The persistent row of a finished model with its batch, hook, test and audit rows. + +use crate::build_progress::_helpers::error_rows::error_detail; +use crate::build_progress::_helpers::model_rows::{ + SubRows, batch_lines, model_line, nested_rows, verbose_blocks, +}; +use crate::build_progress::_helpers::rows::prefix_width; +use crate::build_progress::constants::{FAILED, SUB_INDENT}; +use crate::build_progress::models::{ + ModelResultText, ModelRowFacts, ProgressLayout, TestResultText, +}; +use crate::cli_text::models::CliStyle; + +/// The model row at `counter` and every row nested under it. +#[must_use] +pub fn render_model_result( + (model, facts, tests): (&ModelResultText, &ModelRowFacts, &[TestResultText]), + counter: usize, + layout: ProgressLayout, + style: CliStyle, +) -> String { + let mut text: String = model_line((model, facts), counter, layout, style); + if layout.verbose { + text.push_str(&verbose_blocks(model, facts, style)); + } + text.push_str(&batch_lines(model, style)); + let sub: SubRows = SubRows { + pad: " ".repeat(prefix_width(layout) + SUB_INDENT), + name_width: layout.name_width.saturating_sub(SUB_INDENT), + style, + }; + text.push_str(&nested_rows(model, tests, &sub, layout.verbose)); + if model.status == FAILED + && let Some(message) = model.error.message.as_deref().filter(|m| !m.is_empty()) + { + text.push_str(&error_detail( + (&model.error, message), + (prefix_width(layout), layout.columns), + false, + style, + )); + } + text +} diff --git a/crates/sqlbuild-core/src/build_progress/main/render_resource_result.rs b/crates/sqlbuild-core/src/build_progress/main/render_resource_result.rs new file mode 100644 index 0000000000..6abf363972 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/render_resource_result.rs @@ -0,0 +1,82 @@ +//! The persistent row of a finished seed, function or source load. + +use crate::build_progress::_helpers::error_rows::error_detail; +use crate::build_progress::_helpers::rows::{ + TopLevelRow, event_blocks, prefix_width, top_level_line, +}; +use crate::build_progress::_helpers::text::{ + counter_text, execution_status_display, format_duration, group_thousands, skip_detail, + truncate_name, +}; +use crate::build_progress::constants::{FAILED, SKIPPED}; +use crate::build_progress::models::{ErrorText, ProgressLayout, ResourceResult}; +use crate::cli_text::models::CliStyle; + +/// The row at `counter`, its log and SQL blocks, and its inline error. +#[must_use] +pub fn render_resource_result( + result: ResourceResult<'_>, + counter: usize, + layout: ProgressLayout, + style: CliStyle, +) -> String { + let counter: String = counter_text(counter, layout.total); + match result { + ResourceResult::Node(node) => { + let mut text: String = top_level_line( + &TopLevelRow { + counter: &counter, + resource_type: &node.kind, + name: &truncate_name(&node.name, layout.name_width), + status: &execution_status_display(&node.status), + duration: &format_duration(node.duration_ms), + detail: "", + }, + layout, + style, + ); + if node.status == FAILED + && let Some(message) = node.error.message.as_deref().filter(|m| !m.is_empty()) + { + text.push_str(&error_detail( + (&node.error, message), + (prefix_width(layout), layout.columns), + false, + style, + )); + } + text + } + ResourceResult::Load(load) => { + let detail: String = if load.status == SKIPPED { + skip_detail(&load.skip) + } else { + format!(" rows={}", group_thousands(load.rows_loaded)) + }; + let mut text: String = top_level_line( + &TopLevelRow { + counter: &counter, + resource_type: &load.resource_kind, + name: &truncate_name(&load.source_name, layout.name_width), + status: &execution_status_display(&load.status), + duration: &format_duration(load.duration_ms), + detail: &detail, + }, + layout, + style, + ); + text.push_str(&event_blocks(&load.events, layout.verbose, style)); + if load.status == FAILED + && let Some(message) = load.error_message.as_deref().filter(|m| !m.is_empty()) + { + text.push_str(&error_detail( + (&ErrorText::default(), message), + (prefix_width(layout), layout.columns), + false, + style, + )); + } + text + } + } +} diff --git a/crates/sqlbuild-core/src/build_progress/main/runtime_diagnostic_summary.rs b/crates/sqlbuild-core/src/build_progress/main/runtime_diagnostic_summary.rs new file mode 100644 index 0000000000..bf42ad0e75 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/runtime_diagnostic_summary.rs @@ -0,0 +1,28 @@ +//! The closing lines for scheduler and query diagnostics omitted past their limits. + +use crate::build_progress::constants::{QUERY_DIAGNOSTIC_LIMIT, SCHEDULER_DIAGNOSTIC_LIMIT}; +use crate::build_progress::main::query_diagnostic_message::query_diagnostic_message; +use crate::build_progress::main::scheduler_diagnostic_message::scheduler_diagnostic_message; +use crate::build_progress::models::{SchedulerStateText, StatementText}; + +/// One line per diagnostic kind that omitted entries, naming the last omitted one. +#[must_use] +pub fn runtime_diagnostic_summary( + scheduler: Option<(usize, SchedulerStateText)>, + query: Option<(usize, &StatementText)>, +) -> Vec { + let mut lines: Vec = Vec::new(); + if let Some((omitted, state)) = scheduler.filter(|(omitted, _)| *omitted > 0) { + lines.push(format!( + "Scheduler diagnostics {omitted} transitions omitted after {SCHEDULER_DIAGNOSTIC_LIMIT}; final state: {}", + scheduler_diagnostic_message(state) + )); + } + if let Some((omitted, statement)) = query.filter(|(omitted, _)| *omitted > 0) { + lines.push(format!( + "Query diagnostics {omitted} statements omitted after {QUERY_DIAGNOSTIC_LIMIT}; final query: {}", + query_diagnostic_message(statement) + )); + } + lines +} diff --git a/crates/sqlbuild-core/src/build_progress/main/scheduler_diagnostic_message.rs b/crates/sqlbuild-core/src/build_progress/main/scheduler_diagnostic_message.rs new file mode 100644 index 0000000000..ecb2b673af --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/scheduler_diagnostic_message.rs @@ -0,0 +1,24 @@ +//! The diagnostic line of one concurrent scheduler state transition. + +use crate::build_progress::models::SchedulerStateText; + +/// Running, ready, waiting and aborted counts against the concurrency limit. +#[must_use] +pub fn scheduler_diagnostic_message(state: SchedulerStateText) -> String { + let counts: String = format!( + "Scheduler {} running, {} ready, {} waiting on dependencies", + state.running, state.ready, state.waiting + ); + if state.aborted != 0 { + return format!( + "{counts}, {} aborted after stop, limit {}", + state.aborted, state.limit + ); + } + let suffix: &str = if state.running < state.limit && state.ready == 0 && state.waiting > 0 { + " (DAG frontier constrained)" + } else { + "" + }; + format!("{counts}, limit {}{suffix}", state.limit) +} diff --git a/crates/sqlbuild-core/src/build_progress/mod.rs b/crates/sqlbuild-core/src/build_progress/mod.rs new file mode 100644 index 0000000000..ed82abaa70 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/mod.rs @@ -0,0 +1,9 @@ +//! Live progress rows and the completion footer of `sqb build`. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/build_progress/models.rs b/crates/sqlbuild-core/src/build_progress/models.rs new file mode 100644 index 0000000000..7ba6a5c391 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/models.rs @@ -0,0 +1,211 @@ +//! Plain data of executor results as build progress output renders them. + +/// The coded error recorded on one result. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ErrorText { + pub code: Option, + pub message: Option, + pub help: Option, +} + +/// Why a result was skipped. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SkipText { + pub mode: Option, + pub reason: Option, +} + +/// One lifecycle event of a model or load run. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum LifecycleEventText { + Sql(String), + Log(String), +} + +/// One audit result. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AuditResultText { + pub audit_name: String, + pub attached_column_name: Option, + pub attached_target_name: Option, + pub run_scope_phase: String, + pub outcome: String, + pub row_count: i64, + pub reused: bool, + pub executed_sql: Option, + /// The audit is evaluated as a measurement instead of violation rows. + pub measurement: bool, + /// `str(measured_value)`. + pub measured_value: String, + pub execution_error: Option, +} + +/// One pre- or post-hook result. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct HookResultText { + pub pre_hook: bool, + pub hook_type: String, + pub label: String, + pub status: String, + pub skip: SkipText, +} + +/// One model result. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ModelResultText { + pub name: String, + pub status: String, + pub failed_phase: Option, + pub staging_relation: Option, + pub duration_ms: Option, + pub batch_count: Option, + pub batch_size: Option, + pub rows_affected: Option, + /// The displayed first and inclusive last cursor bound, when both are known. + pub cursor_range: Option<(String, String)>, + pub empty_cursor_inputs: Vec, + pub waiting_on_empty_inputs: bool, + pub recovery_batch_count: i64, + pub synthetic_completion_count: i64, + pub replay_requirement_state: Option, + pub audits: Vec, + pub warnings: Vec, + pub events: Vec, + pub hooks: Vec, + pub skip: SkipText, + pub error: ErrorText, +} + +/// One seed or function result. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct NodeResultText { + pub name: String, + /// `seed`, or the function kind. + pub kind: String, + pub status: String, + pub duration_ms: Option, + pub error: ErrorText, + pub warnings: Vec, +} + +/// One source load result. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct LoadRowText { + pub source_name: String, + pub loader_name: String, + pub resource_kind: String, + pub status: String, + pub rows_loaded: i64, + pub duration_ms: Option, + pub events: Vec, + pub skip: SkipText, + pub error_message: Option, + pub warnings: Vec, +} + +/// One expectation step of a SQL test. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TestStepText { + pub model_name: String, + pub outcome: String, + pub detail: String, +} + +/// One SQL test result. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TestResultText { + /// The test name with its parameter label. + pub label: String, + pub outcome: String, + pub steps: Vec, + pub error: ErrorText, +} + +/// One Python node result. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PythonNodeText { + pub name: String, + pub kind: String, + pub status: String, + pub error_message: Option, + pub warnings: Vec, +} + +/// Every result of one build run. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct BuildResultText { + pub failed: bool, + pub warning_count: i64, + pub insufficient_count: i64, + pub models: Vec, + pub seeds: Vec, + pub functions: Vec, + pub loads: Vec, + pub tests: Vec, + pub source_audits: Vec, + pub end_audits: Vec, +} + +/// The plan facts of the model a row reports. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModelRowFacts { + pub display_type: String, + pub annotation: String, + pub logical_ddl: Option, +} + +/// Fixed geometry of the progress rows of one run. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ProgressLayout { + pub total: usize, + pub name_width: usize, + pub columns: usize, + pub verbose: bool, +} + +/// One concurrent scheduler state transition. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct SchedulerStateText { + pub running: i64, + pub ready: i64, + pub waiting: i64, + pub limit: i64, + pub aborted: i64, +} + +/// One completed warehouse statement. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct StatementText { + pub query_id: Option, + pub status: String, + pub elapsed_seconds: f64, + pub resource_type: String, + pub resource_name: String, + pub phase: String, +} + +/// The node shown on the live spinner row. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SpinnerRow { + /// The 1-based position the running node will complete at. + pub position: usize, + pub resource_kind: String, + pub name: String, + pub message: String, +} + +/// Names that set the shared name column width. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct NameWidthInputs { + /// Top-level row names, with model annotations. + pub names: Vec, + /// Expectation rows drawn as tree children. + pub tree_child_names: Vec, +} + +/// A finished seed or function, or a source load. +#[derive(Clone, Copy, Debug)] +pub enum ResourceResult<'a> { + Node(&'a NodeResultText), + Load(&'a LoadRowText), +} diff --git a/crates/sqlbuild-core/src/build_progress/tests.rs b/crates/sqlbuild-core/src/build_progress/tests.rs new file mode 100644 index 0000000000..58eafc32fe --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_build_progress.rs"] +mod build_progress; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/build_progress/tests/helpers.rs b/crates/sqlbuild-core/src/build_progress/tests/helpers.rs new file mode 100644 index 0000000000..68ea5bff0f --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/tests/helpers.rs @@ -0,0 +1,57 @@ +use crate::build_progress::models::{ + AuditResultText, ErrorText, ModelResultText, ProgressLayout, TestResultText, TestStepText, +}; + +pub(super) fn layout(verbose: bool) -> ProgressLayout { + ProgressLayout { + total: 3, + name_width: 24, + columns: 80, + verbose, + } +} + +pub(super) fn model(name: &str, status: &str) -> ModelResultText { + ModelResultText { + name: name.to_owned(), + status: status.to_owned(), + duration_ms: Some(1234.0), + ..ModelResultText::default() + } +} + +pub(super) fn failed_model(name: &str, message: &str) -> ModelResultText { + ModelResultText { + failed_phase: Some("staging".to_owned()), + error: ErrorText { + code: Some("E100".to_owned()), + message: Some(message.to_owned()), + help: None, + }, + ..model(name, "failed") + } +} + +pub(super) fn audit(name: &str, outcome: &str, row_count: i64) -> AuditResultText { + AuditResultText { + audit_name: name.to_owned(), + run_scope_phase: "final".to_owned(), + outcome: outcome.to_owned(), + row_count, + measured_value: "None".to_owned(), + ..AuditResultText::default() + } +} + +pub(super) fn test(label: &str, outcome: &str, step_outcome: &str) -> TestResultText { + TestResultText { + label: label.to_owned(), + outcome: outcome.to_owned(), + steps: vec![TestStepText { + model_name: "orders".to_owned(), + outcome: step_outcome.to_owned(), + detail: String::new(), + }], + error: ErrorText::default(), + } +} diff --git a/crates/sqlbuild-core/src/build_progress/tests/test_build_progress.rs b/crates/sqlbuild-core/src/build_progress/tests/test_build_progress.rs new file mode 100644 index 0000000000..1fac3f177c --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/tests/test_build_progress.rs @@ -0,0 +1,295 @@ +use crate::build_progress::main::format_build_footer::format_build_footer; +use crate::build_progress::main::progress_name_width::progress_name_width; +use crate::build_progress::main::render_build_spinner_line::render_build_spinner_line; +use crate::build_progress::main::render_failed_test_result::render_failed_test_result; +use crate::build_progress::main::render_model_result::render_model_result; +use crate::build_progress::main::scheduler_diagnostic_message::scheduler_diagnostic_message; +use crate::build_progress::models::{ + BuildResultText, ErrorText, LifecycleEventText, ModelResultText, ModelRowFacts, + NameWidthInputs, ProgressLayout, PythonNodeText, SchedulerStateText, SpinnerRow, +}; +use crate::build_progress::tests::helpers::{audit, failed_model, layout, model, test}; +use crate::build_progress::tests::test_types::{ + FailedTestTestCase, FooterTestCase, InlineErrorTestCase, ModelRowTestCase, NameWidthTestCase, + SchedulerTestCase, SpinnerTestCase, +}; +use crate::cli_text::main::format_inline_error_lines::format_inline_error_lines; +use crate::cli_text::models::CliStyle; + +const PLAIN: CliStyle = CliStyle { use_color: false }; + +#[test] +fn given_build_results_when_formatting_footer_then_matches_python_layout() { + let test_cases = [ + FooterTestCase { + description: "a clean run reports success", + result: BuildResultText { + models: vec![model("orders", "success")], + ..BuildResultText::default() + }, + python: Vec::new(), + use_color: false, + expected_text: "\u{2713} Completed successfully PASS=1 WARN=0 FAIL=0 INSUFFICIENT=0 SKIP=0 TOTAL=1 (1.50s)", + }, + FooterTestCase { + description: "warnings, failures and skipped nodes each get a section", + result: BuildResultText { + failed: true, + models: vec![ + ModelResultText { + audits: vec![audit("not_null", "warn", 2)], + ..model("orders", "success") + }, + ModelResultText { + staging_relation: Some("orders__delta".to_owned()), + ..failed_model("customers", "boom") + }, + model("payments", "skipped"), + ], + ..BuildResultText::default() + }, + python: vec![PythonNodeText { + name: "refresh".to_owned(), + kind: "task".to_owned(), + status: "failed".to_owned(), + error_message: None, + warnings: Vec::new(), + }], + use_color: false, + expected_text: "\nWarnings:\n\n orders\n audit not_null returned 2 rows\n\n\u{2717} Completed with errors PASS=1 WARN=1 FAIL=2 INSUFFICIENT=0 SKIP=1 TOTAL=5 (1.50s)\n\nFailures:\n\n customers (staging)\n error [E100] boom\n delta table kept for inspection: orders__delta\n\n\nPython node failures:\n\n task refresh Python node failed\n\nSkipped (1): payments", + }, + ]; + + for test_case in test_cases { + let text: String = format_build_footer( + &test_case.result, + &test_case.python, + 1.5, + CliStyle { + use_color: test_case.use_color, + }, + ); + assert_eq!(text, test_case.expected_text, "{}", test_case.description); + } +} + +#[test] +fn given_model_result_when_rendering_then_nests_tests_audits_and_errors() { + let test_cases = [ + ModelRowTestCase { + description: "a passing model lists its tests and aggregated audits", + model: ModelResultText { + audits: vec![audit("unique", "error", 1), audit("unique", "pass", 0)], + ..model("orders", "success") + }, + tests: vec![test("orders_test", "pass", "pass")], + verbose: false, + expected_text: " 2/3 table orders OK 1.23s\n test orders_test PASS\n expect expected orders PASS\n audit unique FAIL 1/2 1 row\n", + }, + ModelRowTestCase { + description: "a failed model shows its phase and wrapped coded error", + model: failed_model("customers", "boom"), + tests: Vec::new(), + verbose: false, + expected_text: " 2/3 table customers FAIL 1.23s staging\n error [E100] boom\n", + }, + ModelRowTestCase { + description: "verbose output prints lifecycle SQL with a semicolon", + model: ModelResultText { + events: vec![LifecycleEventText::Sql("select 1 ".to_owned())], + batch_count: Some(2), + rows_affected: Some(12_345), + ..model("orders", "success") + }, + tests: Vec::new(), + verbose: true, + expected_text: " 2/3 table orders OK 1.23s\n\n select 1;\n\n 2 batches 12.3K rows\n", + }, + ]; + + for test_case in test_cases { + let facts: ModelRowFacts = ModelRowFacts { + display_type: "table".to_owned(), + ..ModelRowFacts::default() + }; + assert_eq!( + render_model_result( + (&test_case.model, &facts, &test_case.tests), + 2, + layout(test_case.verbose), + PLAIN + ), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_failed_test_when_rendering_then_draws_tree_children() { + let test_cases = [ + FailedTestTestCase { + description: "the last expectation closes the tree", + test: test("orders_test", "fail", "fail"), + expected_text: " test orders_test FAIL\n \u{2514}\u{2500}\u{2500} expect expected orders FAIL\n", + }, + FailedTestTestCase { + description: "an error closes the tree after the expectations", + test: crate::build_progress::models::TestResultText { + error: ErrorText { + code: None, + message: Some("broken".to_owned()), + help: None, + }, + ..test("orders_test", "error", "error") + }, + expected_text: " test orders_test ERROR\n \u{251c}\u{2500}\u{2500} expect expected orders ERROR\n \u{2514}\u{2500}\u{2500} error broken\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_failed_test_result(&test_case.test, layout(false), PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_running_node_when_drawing_spinner_then_fits_terminal() { + let test_cases = [ + SpinnerTestCase { + description: "a wide terminal shows the full row", + row: SpinnerRow { + position: 1, + resource_kind: "table".to_owned(), + name: "orders".to_owned(), + message: "batch 1/2".to_owned(), + }, + columns: 120, + expected_text: "\r\x1b[K 1/3 table orders batch 1/2 \u{280b}", + }, + SpinnerTestCase { + description: "a narrow terminal truncates the row", + row: SpinnerRow { + position: 1, + resource_kind: "table".to_owned(), + name: "orders".to_owned(), + message: String::new(), + }, + columns: 24, + expected_text: "\r\x1b[K 1/3 table o... \u{280b}", + }, + ]; + + for test_case in test_cases { + let layout: ProgressLayout = ProgressLayout { + columns: test_case.columns, + ..layout(false) + }; + assert_eq!( + render_build_spinner_line(&test_case.row, 0, layout, PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_row_names_when_sizing_then_tree_children_keep_status_column() { + let test_cases = [ + NameWidthTestCase { + description: "short names use the minimum width", + inputs: NameWidthInputs { + names: vec!["orders".to_owned()], + tree_child_names: Vec::new(), + }, + expected_width: 20, + }, + NameWidthTestCase { + description: "expectation children add their tree indent", + inputs: NameWidthInputs { + names: vec!["orders".to_owned()], + tree_child_names: vec!["a_rather_long_model_name".to_owned()], + }, + expected_width: 39, + }, + ]; + + for test_case in test_cases { + assert_eq!( + progress_name_width(&test_case.inputs), + test_case.expected_width, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_scheduler_state_when_describing_then_flags_frontier_constraints() { + let test_cases = [ + SchedulerTestCase { + description: "an idle worker with only waiting nodes is frontier constrained", + state: SchedulerStateText { + running: 1, + ready: 0, + waiting: 3, + limit: 4, + aborted: 0, + }, + expected_text: "Scheduler 1 running, 0 ready, 3 waiting on dependencies, limit 4 (DAG frontier constrained)", + }, + SchedulerTestCase { + description: "aborted nodes are reported after a stop", + state: SchedulerStateText { + running: 0, + ready: 1, + waiting: 2, + limit: 4, + aborted: 5, + }, + expected_text: "Scheduler 0 running, 1 ready, 2 waiting on dependencies, 5 aborted after stop, limit 4", + }, + ]; + + for test_case in test_cases { + assert_eq!( + scheduler_diagnostic_message(test_case.state), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_long_error_when_wrapping_inline_then_truncates_to_two_lines() { + let test_cases = [ + InlineErrorTestCase { + description: "a coded error with help wraps and keeps the code first", + error: (Some("E1"), "short", Some("do this")), + width: 40, + expected_lines: &["[E1] short", "= help: do this"], + }, + InlineErrorTestCase { + description: "hyphenated words are not split and overflow is truncated", + error: (None, "alpha-beta-gamma delta epsilon zeta eta", None), + width: 12, + expected_lines: &["alpha-beta-g", "amma delt..."], + }, + ]; + + for test_case in test_cases { + assert_eq!( + format_inline_error_lines(test_case.error, test_case.width, PLAIN), + test_case.expected_lines, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/build_progress/tests/test_types.rs b/crates/sqlbuild-core/src/build_progress/tests/test_types.rs new file mode 100644 index 0000000000..a258d7dd49 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/tests/test_types.rs @@ -0,0 +1,52 @@ +use crate::build_progress::models::{ + BuildResultText, ModelResultText, NameWidthInputs, PythonNodeText, SchedulerStateText, + SpinnerRow, TestResultText, +}; + +pub(super) struct FooterTestCase { + pub(super) description: &'static str, + pub(super) result: BuildResultText, + pub(super) python: Vec, + pub(super) use_color: bool, + pub(super) expected_text: &'static str, +} + +pub(super) struct ModelRowTestCase { + pub(super) description: &'static str, + pub(super) model: ModelResultText, + pub(super) tests: Vec, + pub(super) verbose: bool, + pub(super) expected_text: &'static str, +} + +pub(super) struct FailedTestTestCase { + pub(super) description: &'static str, + pub(super) test: TestResultText, + pub(super) expected_text: &'static str, +} + +pub(super) struct SpinnerTestCase { + pub(super) description: &'static str, + pub(super) row: SpinnerRow, + pub(super) columns: usize, + pub(super) expected_text: &'static str, +} + +pub(super) struct NameWidthTestCase { + pub(super) description: &'static str, + pub(super) inputs: NameWidthInputs, + pub(super) expected_width: usize, +} + +pub(super) struct SchedulerTestCase { + pub(super) description: &'static str, + pub(super) state: SchedulerStateText, + pub(super) expected_text: &'static str, +} + +pub(super) struct InlineErrorTestCase { + pub(super) description: &'static str, + pub(super) error: (Option<&'static str>, &'static str, Option<&'static str>), + pub(super) width: usize, + pub(super) expected_lines: &'static [&'static str], +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/coded_error.rs b/crates/sqlbuild-core/src/cli_text/_helpers/coded_error.rs index b80362ec49..fc0695668a 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/coded_error.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/coded_error.rs @@ -1,6 +1,10 @@ -//! A result-stored coded error, as CLI expected errors render it. +//! Result-stored coded errors, as CLI expected errors and compact inline rows render them. +use crate::cli_text::constants::MAX_INLINE_ERROR_LINES; use crate::cli_text::models::CliStyle; +use crate::text::main::python_isspace::python_isspace; +use crate::text::main::python_splitlines::python_splitlines; +use crate::text::main::text_wrap::text_wrap; /// `error[CODE]: message` or `[CODE] message`, later lines muted, then any help line. pub(crate) fn coded_error( @@ -29,3 +33,63 @@ pub(crate) fn coded_error( } rendered } + +/// `[code] message`, with `= help: help` on its own paragraph. +pub(crate) fn inline_error_text(code_prefix: &str, message: &str, help: Option<&str>) -> String { + let text: String = if code_prefix.is_empty() { + message.to_owned() + } else { + format!("{code_prefix} {message}") + }; + match help { + Some(help) => format!("{text}\n= help: {help}"), + None => text, + } +} + +/// Every paragraph wrapped on whitespace only, an empty paragraph kept as one empty line. +pub(crate) fn wrap_paragraphs(text: &str, width: usize) -> Vec { + let mut paragraphs: Vec<&str> = python_splitlines(text); + if paragraphs.is_empty() { + paragraphs.push(text); + } + let mut wrapped: Vec = Vec::new(); + for paragraph in paragraphs { + let lines: Vec = text_wrap(paragraph, width, ("", ""), false); + if lines.is_empty() { + wrapped.push(String::new()); + } else { + wrapped.extend(lines); + } + } + wrapped +} + +/// The first lines, the last one cut to end in an ellipsis when lines were dropped. +pub(crate) fn truncate_lines(mut lines: Vec, width: usize) -> Vec { + let truncated: bool = lines.len() > MAX_INLINE_ERROR_LINES; + lines.truncate(MAX_INLINE_ERROR_LINES); + if truncated && let Some(last) = lines.last_mut() { + let ellipsis: String = ".".repeat(width.min(3)); + let kept: String = last + .chars() + .take(width.saturating_sub(ellipsis.len())) + .collect(); + *last = format!("{}{ellipsis}", kept.trim_end_matches(python_isspace)); + } + lines +} + +/// The first occurrence of the code prefix in the first line, styled as strong error text. +pub(crate) fn style_code_prefix( + mut lines: Vec, + code_prefix: &str, + style: CliStyle, +) -> Vec { + if !code_prefix.is_empty() + && let Some(first) = lines.first_mut() + { + *first = first.replacen(code_prefix, &style.error_strong(code_prefix), 1); + } + lines +} diff --git a/crates/sqlbuild-core/src/cli_text/constants.rs b/crates/sqlbuild-core/src/cli_text/constants.rs index 6e575a762f..e84eb2526f 100644 --- a/crates/sqlbuild-core/src/cli_text/constants.rs +++ b/crates/sqlbuild-core/src/cli_text/constants.rs @@ -87,3 +87,5 @@ pub(crate) const COUNT_HEADER_OPENING: &str = " ("; pub(crate) const COUNT_HEADER_CLOSING: &str = ")"; pub(crate) const DBT_NO_MATCH_WARNING: &str = "No dbt or SQLBuild resources matched the selection."; pub(crate) const DBT_RESOURCE_FALLBACK_TYPE: &str = "resource"; +/// An inline error wraps to at most this many lines before it is truncated. +pub(crate) const MAX_INLINE_ERROR_LINES: usize = 2; diff --git a/crates/sqlbuild-core/src/cli_text/main/format_inline_error_lines.rs b/crates/sqlbuild-core/src/cli_text/main/format_inline_error_lines.rs new file mode 100644 index 0000000000..f8d9878a04 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_inline_error_lines.rs @@ -0,0 +1,21 @@ +//! A compact inline diagnostic wrapped to two display lines. + +use crate::cli_text::_helpers::coded_error::{ + inline_error_text, style_code_prefix, truncate_lines, wrap_paragraphs, +}; +use crate::cli_text::models::CliStyle; + +/// The `[code] message` and `= help:` text wrapped to two lines, truncated with an ellipsis. +#[must_use] +pub fn format_inline_error_lines( + (code, message, help): (Option<&str>, &str, Option<&str>), + content_width: usize, + style: CliStyle, +) -> Vec { + let code_prefix: String = code.map_or_else(String::new, |code| format!("[{code}]")); + let wrapped: Vec = wrap_paragraphs( + &inline_error_text(&code_prefix, message, help), + content_width, + ); + style_code_prefix(truncate_lines(wrapped, content_width), &code_prefix, style) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/mod.rs b/crates/sqlbuild-core/src/cli_text/main/mod.rs index 8208efb9bd..60ec06a502 100644 --- a/crates/sqlbuild-core/src/cli_text/main/mod.rs +++ b/crates/sqlbuild-core/src/cli_text/main/mod.rs @@ -6,6 +6,7 @@ pub mod format_completion_line; pub mod format_count_noun; pub mod format_dbt_interop_plan; pub mod format_dbt_interop_plan_json; +pub mod format_inline_error_lines; pub mod format_overflow_line; pub mod format_phase_line; pub mod format_plan_completion; diff --git a/crates/sqlbuild-core/src/format_command/_helpers/diagnostics.rs b/crates/sqlbuild-core/src/format_command/_helpers/diagnostics.rs index 5a98a55555..aed12e2d2f 100644 --- a/crates/sqlbuild-core/src/format_command/_helpers/diagnostics.rs +++ b/crates/sqlbuild-core/src/format_command/_helpers/diagnostics.rs @@ -67,6 +67,7 @@ fn help_lines(remediation: &str, style: CliStyle) -> Vec { remediation, REPORT_LINE_WIDTH, (HELP_PREFIX, HELP_CONTINUATION), + true, ); let Some((first, rest)) = wrapped.split_first() else { return Vec::new(); diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index e143b3605c..a363d58d5c 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -4,6 +4,7 @@ pub mod audit_command; pub mod build_command; +pub mod build_progress; pub mod check_command; pub mod cli_text; pub mod clone_text; @@ -25,6 +26,7 @@ pub mod janitor_text; pub mod json; pub mod load_command; pub mod models; +pub mod nested_progress; pub mod panics; pub mod plan_display; pub mod posix_path; diff --git a/crates/sqlbuild-core/src/nested_progress/constants.rs b/crates/sqlbuild-core/src/nested_progress/constants.rs new file mode 100644 index 0000000000..bdddc490b7 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/constants.rs @@ -0,0 +1,13 @@ +//! Column widths of grouped command progress rows. + +pub(crate) const LABEL_WIDTH: usize = 10; +pub(crate) const CHILD_NAME_INDENT: usize = 4; +pub(crate) const LIVE_ROW_PADDING: usize = 7; +pub(crate) const AUDIT_NAME_WIDTH: usize = 50; +pub(crate) const ERROR_PAD: &str = " "; +pub(crate) const ERROR_CONNECTOR_WIDTH: usize = 4; +pub(crate) const UNATTACHED_GROUP: &str = "(unattached)"; +pub(crate) const CLEAR_LINE: &str = "\r\x1b[K"; +pub(crate) const PASS_OUTCOME: &str = "pass"; +pub(crate) const WARN_OUTCOME: &str = "warn"; +pub(crate) const ERROR_OUTCOME: &str = "error"; diff --git a/crates/sqlbuild-core/src/nested_progress/main/audit_group_name.rs b/crates/sqlbuild-core/src/nested_progress/main/audit_group_name.rs new file mode 100644 index 0000000000..9a20999d31 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/main/audit_group_name.rs @@ -0,0 +1,13 @@ +//! The group an audit result row is listed under. + +use crate::build_progress::models::AuditResultText; +use crate::nested_progress::constants::UNATTACHED_GROUP; + +/// The attached target, or `(unattached)`. +#[must_use] +pub fn audit_group_name(result: &AuditResultText) -> String { + match result.attached_target_name.as_deref() { + Some(target) if !target.is_empty() => target.to_owned(), + _ => UNATTACHED_GROUP.to_owned(), + } +} diff --git a/crates/sqlbuild-core/src/nested_progress/main/audit_resource_id.rs b/crates/sqlbuild-core/src/nested_progress/main/audit_resource_id.rs new file mode 100644 index 0000000000..41faf029a7 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/main/audit_resource_id.rs @@ -0,0 +1,17 @@ +//! The execution-protocol identity of one attached audit. + +/// `audit::::`, without empty parts. +#[must_use] +pub fn audit_resource_id( + audit_name: &str, + identity_kind: &str, + (target, column): (Option<&str>, Option<&str>), +) -> String { + ["audit", audit_name, identity_kind] + .into_iter() + .chain(target) + .chain(column) + .filter(|part| !part.is_empty()) + .collect::>() + .join(":") +} diff --git a/crates/sqlbuild-core/src/nested_progress/main/mod.rs b/crates/sqlbuild-core/src/nested_progress/main/mod.rs new file mode 100644 index 0000000000..1f0a78ad20 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/main/mod.rs @@ -0,0 +1,7 @@ +pub mod audit_group_name; +pub mod audit_resource_id; +pub mod render_audit_aggregate; +pub mod render_audit_row; +pub mod render_group_header; +pub mod render_nested_item; +pub mod render_nested_spinner_line; diff --git a/crates/sqlbuild-core/src/nested_progress/main/render_audit_aggregate.rs b/crates/sqlbuild-core/src/nested_progress/main/render_audit_aggregate.rs new file mode 100644 index 0000000000..9bf29b7368 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/main/render_audit_aggregate.rs @@ -0,0 +1,13 @@ +//! The live aggregate row of concurrently running audits. + +use crate::nested_progress::constants::CLEAR_LINE; +use crate::nested_progress::models::AuditAggregate; + +/// Running, completed and worker counts on one cleared line. +#[must_use] +pub fn render_audit_aggregate(aggregate: AuditAggregate) -> String { + format!( + "{CLEAR_LINE}{CLEAR_LINE} audit running {} | completed {}/{} | workers {}", + aggregate.running, aggregate.completed, aggregate.total, aggregate.workers + ) +} diff --git a/crates/sqlbuild-core/src/nested_progress/main/render_audit_row.rs b/crates/sqlbuild-core/src/nested_progress/main/render_audit_row.rs new file mode 100644 index 0000000000..3b9430ebeb --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/main/render_audit_row.rs @@ -0,0 +1,39 @@ +//! One completed standalone audit row. + +use crate::build_progress::models::AuditResultText; +use crate::cli_text::models::CliStyle; +use crate::nested_progress::constants::{ + AUDIT_NAME_WIDTH, ERROR_OUTCOME, LABEL_WIDTH, PASS_OUTCOME, WARN_OUTCOME, +}; + +/// The audit name and column, status, failing row count and optional duration. +#[must_use] +pub fn render_audit_row( + result: &AuditResultText, + duration_ms: Option, + style: CliStyle, +) -> String { + let status_text: &str = match result.outcome.as_str() { + PASS_OUTCOME => "PASS", + WARN_OUTCOME => "WARN", + ERROR_OUTCOME => "FAIL", + _ => "INSUFFICIENT", + }; + let name: String = match &result.attached_column_name { + Some(column) => format!("{} ({column})", result.audit_name), + None => result.audit_name.clone(), + }; + let mut detail: String = String::new(); + if result.outcome != PASS_OUTCOME && result.row_count > 0 { + let label: &str = if result.row_count == 1 { "row" } else { "rows" }; + detail = format!(" {} {label}", result.row_count); + } + if let Some(duration_ms) = duration_ms { + detail = format!("{detail} {:.2}s", duration_ms / 1000.0); + } + format!( + " {: String { + let prefix: &str = if first { "" } else { "\n" }; + format!("{prefix}{}\n", style.section(group_name)) +} diff --git a/crates/sqlbuild-core/src/nested_progress/main/render_nested_item.rs b/crates/sqlbuild-core/src/nested_progress/main/render_nested_item.rs new file mode 100644 index 0000000000..24e338ca69 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/main/render_nested_item.rs @@ -0,0 +1,59 @@ +//! A completed grouped item with its tree of child rows and inline error. + +use crate::cli_text::main::format_inline_error_lines::format_inline_error_lines; +use crate::cli_text::main::format_tree_connector::format_tree_connector; +use crate::cli_text::models::CliStyle; +use crate::nested_progress::constants::{ + CHILD_NAME_INDENT, ERROR_CONNECTOR_WIDTH, ERROR_PAD, LABEL_WIDTH, +}; +use crate::nested_progress::models::{NestedItem, NestedLayout}; + +/// The item row, one tree row per child, and the error as the last tree child. +#[must_use] +pub fn render_nested_item(item: &NestedItem, layout: &NestedLayout, style: CliStyle) -> String { + let width: usize = layout.name_width; + let mut text: String = format!( + " {: = format_inline_error_lines( + ( + item.error.code.as_deref(), + message, + item.error.help.as_deref(), + ), + content_width, + style, + ); + let connector: String = format_tree_connector(style, true); + for (index, line) in lines.iter().enumerate() { + if index == 0 { + text.push_str(&format!("{ERROR_PAD}{connector} {label}{line}\n")); + } else { + text.push_str(&format!("{}{line}\n", " ".repeat(prefix_width))); + } + } + text +} diff --git a/crates/sqlbuild-core/src/nested_progress/main/render_nested_spinner_line.rs b/crates/sqlbuild-core/src/nested_progress/main/render_nested_spinner_line.rs new file mode 100644 index 0000000000..5b8a8d55e7 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/main/render_nested_spinner_line.rs @@ -0,0 +1,36 @@ +//! The single live row naming every running item. + +use crate::cli_text::models::CliStyle; +use crate::nested_progress::constants::{CLEAR_LINE, LABEL_WIDTH, LIVE_ROW_PADDING}; +use crate::nested_progress::models::NestedLayout; + +/// The running item, or a count and list of running items, cut to the terminal. +#[must_use] +pub fn render_nested_spinner_line( + running: &[String], + frame: &str, + layout: &NestedLayout, + style: CliStyle, +) -> String { + let text: String = match running { + [only] => only.clone(), + _ => format!("{} running: {}", running.len(), running.join(", ")), + }; + let available: i64 = layout.columns as i64 - (LABEL_WIDTH + LIVE_ROW_PADDING) as i64; + let width: usize = (layout.name_width as i64).min(available).max(1) as usize; + let name: String = if text.chars().count() <= width { + text + } else { + format!( + "{}...", + text.chars() + .take(width.saturating_sub(3)) + .collect::() + ) + }; + format!( + "{CLEAR_LINE} {:, +} + +/// The row label, name column width and terminal width of one command's rows. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct NestedLayout { + pub label: String, + pub name_width: usize, + pub columns: usize, +} + +/// Aggregate worker state of concurrently running audits. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct AuditAggregate { + pub running: usize, + pub completed: usize, + pub total: usize, + pub workers: i64, +} diff --git a/crates/sqlbuild-core/src/nested_progress/tests.rs b/crates/sqlbuild-core/src/nested_progress/tests.rs new file mode 100644 index 0000000000..718cf08381 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_nested_progress.rs"] +mod nested_progress; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/nested_progress/tests/helpers.rs b/crates/sqlbuild-core/src/nested_progress/tests/helpers.rs new file mode 100644 index 0000000000..00b9ee32fa --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/tests/helpers.rs @@ -0,0 +1,9 @@ +use crate::nested_progress::models::NestedLayout; + +pub(super) fn layout(columns: usize) -> NestedLayout { + NestedLayout { + label: "test".to_owned(), + name_width: 20, + columns, + } +} diff --git a/crates/sqlbuild-core/src/nested_progress/tests/test_nested_progress.rs b/crates/sqlbuild-core/src/nested_progress/tests/test_nested_progress.rs new file mode 100644 index 0000000000..73b8a77bf5 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/tests/test_nested_progress.rs @@ -0,0 +1,87 @@ +use crate::build_progress::models::ErrorText; +use crate::cli_text::models::CliStyle; +use crate::nested_progress::main::render_nested_item::render_nested_item; +use crate::nested_progress::main::render_nested_spinner_line::render_nested_spinner_line; +use crate::nested_progress::models::{NestedChildRow, NestedItem}; +use crate::nested_progress::tests::helpers::layout; +use crate::nested_progress::tests::test_types::{NestedItemTestCase, SpinnerTestCase}; + +const PLAIN: CliStyle = CliStyle { use_color: false }; + +#[test] +fn given_completed_item_when_rendering_then_draws_children_and_error_as_tree() { + let test_cases = [ + NestedItemTestCase { + description: "the last child closes the tree", + item: NestedItem { + item_name: "orders_test".to_owned(), + status_text: "PASS".to_owned(), + detail: " 0.10s".to_owned(), + error: ErrorText::default(), + child_rows: vec![NestedChildRow { + label: "expect".to_owned(), + name: "expected orders".to_owned(), + status_text: "PASS".to_owned(), + detail: String::new(), + }], + }, + expected_text: " test orders_test PASS 0.10s\n \u{2514}\u{2500}\u{2500} expect expected orders PASS\n", + }, + NestedItemTestCase { + description: "an error is the final tree child", + item: NestedItem { + item_name: "orders_test".to_owned(), + status_text: "ERROR".to_owned(), + detail: String::new(), + error: ErrorText { + code: Some("T1".to_owned()), + message: Some("broken".to_owned()), + help: None, + }, + child_rows: Vec::new(), + }, + expected_text: " test orders_test ERROR\n \u{2514}\u{2500}\u{2500} error [T1] broken\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_nested_item(&test_case.item, &layout(80), PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_running_items_when_drawing_spinner_then_lists_and_truncates() { + let test_cases = [ + SpinnerTestCase { + description: "one running item is shown by name", + running: vec!["orders_test".to_owned()], + columns: 80, + expected_text: "\r\x1b[K test orders_test \u{280b}", + }, + SpinnerTestCase { + description: "several running items are counted and cut to the terminal", + running: vec!["orders_test".to_owned(), "customers_test".to_owned()], + columns: 30, + expected_text: "\r\x1b[K test 2 running:... \u{280b}", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_nested_spinner_line( + &test_case.running, + "\u{280b}", + &layout(test_case.columns), + PLAIN + ), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/nested_progress/tests/test_types.rs b/crates/sqlbuild-core/src/nested_progress/tests/test_types.rs new file mode 100644 index 0000000000..41d8d56d84 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/tests/test_types.rs @@ -0,0 +1,14 @@ +use crate::nested_progress::models::NestedItem; + +pub(super) struct NestedItemTestCase { + pub(super) description: &'static str, + pub(super) item: NestedItem, + pub(super) expected_text: &'static str, +} + +pub(super) struct SpinnerTestCase { + pub(super) description: &'static str, + pub(super) running: Vec, + pub(super) columns: usize, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-core/src/progress/constants.rs b/crates/sqlbuild-core/src/progress/constants.rs index 4338385c18..aa0ba9ec5e 100644 --- a/crates/sqlbuild-core/src/progress/constants.rs +++ b/crates/sqlbuild-core/src/progress/constants.rs @@ -108,3 +108,5 @@ pub(crate) const DBT_COMMAND: &str = "dbt"; pub(crate) const DBT_JSON_OPTION: &str = "--json"; pub(crate) const JSON_FORMAT: &str = "json"; pub(crate) const CSV_FORMAT: &str = "csv"; +/// What replaces the cut tail of a transient status message. +pub(crate) const STATUS_ELLIPSIS: &str = "..."; diff --git a/crates/sqlbuild-core/src/progress/main/format_status_spinner_line.rs b/crates/sqlbuild-core/src/progress/main/format_status_spinner_line.rs new file mode 100644 index 0000000000..31dc33f57e --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/format_status_spinner_line.rs @@ -0,0 +1,27 @@ +//! The live line of a transient status: a spinner frame and the current message. + +use crate::cli_text::models::CliStyle; +use crate::progress::constants::STATUS_ELLIPSIS; + +/// The frame and the first message line, cut to fit `columns` (`0` leaves it uncut). +#[must_use] +pub fn format_status_spinner_line( + frame: &str, + message: &str, + columns: usize, + style: CliStyle, +) -> String { + let text: &str = message.lines().next().unwrap_or(""); + let width: usize = columns.saturating_sub(frame.chars().count() + 2); + let fitted: String = if columns == 0 || text.chars().count() <= width { + text.to_owned() + } else { + format!( + "{}{STATUS_ELLIPSIS}", + text.chars() + .take(width.saturating_sub(STATUS_ELLIPSIS.len())) + .collect::() + ) + }; + format!("{} {fitted}", style.success(frame)) +} diff --git a/crates/sqlbuild-core/src/progress/main/mod.rs b/crates/sqlbuild-core/src/progress/main/mod.rs index 67bb535223..9b70ca7b90 100644 --- a/crates/sqlbuild-core/src/progress/main/mod.rs +++ b/crates/sqlbuild-core/src/progress/main/mod.rs @@ -8,6 +8,7 @@ pub mod format_execution_header; pub mod format_expectation_detail; pub mod format_expectation_name; pub mod format_inspection_record; +pub mod format_status_spinner_line; pub mod is_planning_completion_message; pub mod machine_output_mode; pub mod progress_terminal_duration; diff --git a/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs b/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs index c6c45ce34d..4ef6de339e 100644 --- a/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs +++ b/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs @@ -5,6 +5,7 @@ use crate::progress::main::execution_header_text::execution_header_text; use crate::progress::main::format_expectation_detail::format_expectation_detail; use crate::progress::main::format_expectation_name::format_expectation_name; use crate::progress::main::format_inspection_record::format_inspection_record; +use crate::progress::main::format_status_spinner_line::format_status_spinner_line; use crate::progress::main::is_planning_completion_message::is_planning_completion_message; use crate::progress::main::machine_output_mode::machine_output_mode; use crate::progress::models::{ExpectationStep, InspectionRecord, OutputModeFacts}; @@ -12,7 +13,7 @@ use crate::progress::models::{ExpectationStep, InspectionRecord, OutputModeFacts const PLAIN: CliStyle = CliStyle { use_color: false }; use crate::progress::tests::test_types::{ CompletionMessageTestCase, ExpectationDetailTestCase, InspectionRecordTestCase, - OutputModeTestCase, ProgressLineTestCase, + OutputModeTestCase, ProgressLineTestCase, StatusSpinnerLineTestCase, }; #[test] @@ -253,3 +254,44 @@ fn given_parsed_options_when_choosing_output_mode_then_machine_output_is_detecte ); } } + +#[test] +fn given_status_message_when_rendering_spinner_line_then_fits_terminal() { + let test_cases = [ + StatusSpinnerLineTestCase { + description: "short message is kept with a green frame", + message: "Compiling project graph...", + columns: 80, + style: CliStyle { use_color: true }, + expected_line: "\x1b[32m\u{280b}\x1b[0m Compiling project graph...", + }, + StatusSpinnerLineTestCase { + description: "long message is cut below the terminal width", + message: "Compiling project graph...", + columns: 15, + style: CliStyle { use_color: false }, + expected_line: "\u{280b} Compiling...", + }, + StatusSpinnerLineTestCase { + description: "unknown width keeps only the first line", + message: "first line\nsecond line", + columns: 0, + style: CliStyle { use_color: false }, + expected_line: "\u{280b} first line", + }, + ]; + + for test_case in test_cases { + assert_eq!( + format_status_spinner_line( + "\u{280b}", + test_case.message, + test_case.columns, + test_case.style + ), + test_case.expected_line, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/progress/tests/test_types.rs b/crates/sqlbuild-core/src/progress/tests/test_types.rs index 48942a07ce..0a9e9d687d 100644 --- a/crates/sqlbuild-core/src/progress/tests/test_types.rs +++ b/crates/sqlbuild-core/src/progress/tests/test_types.rs @@ -1,3 +1,4 @@ +use crate::cli_text::models::CliStyle; use crate::progress::models::{ProgressEvent, ProgressSettings}; use crate::terminal_events::models::TerminalRequest; @@ -40,3 +41,11 @@ pub(super) struct OutputModeTestCase { pub(super) facts: crate::progress::models::OutputModeFacts, pub(super) expected_machine_output: bool, } + +pub(super) struct StatusSpinnerLineTestCase { + pub(super) description: &'static str, + pub(super) message: &'static str, + pub(super) columns: usize, + pub(super) style: CliStyle, + pub(super) expected_line: &'static str, +} diff --git a/crates/sqlbuild-core/src/text/_helpers/text_wrap.rs b/crates/sqlbuild-core/src/text/_helpers/text_wrap.rs index 28ed46aa88..e9ac1b13d5 100644 --- a/crates/sqlbuild-core/src/text/_helpers/text_wrap.rs +++ b/crates/sqlbuild-core/src/text/_helpers/text_wrap.rs @@ -32,10 +32,19 @@ fn munge_whitespace(text: &str) -> Vec { munged } -/// `textwrap.wrap(text, width, initial_indent=..., subsequent_indent=...)`. -pub(crate) fn wrap(text: &str, width: usize, (initial, subsequent): (&str, &str)) -> Vec { +/// `textwrap.wrap(text, width, initial_indent=..., subsequent_indent=..., break_on_hyphens=...)`. +pub(crate) fn wrap( + text: &str, + width: usize, + (initial, subsequent): (&str, &str), + break_on_hyphens: bool, +) -> Vec { let characters: Vec = munge_whitespace(text); - let mut chunks: Vec> = split_chunks(&characters); + let mut chunks: Vec> = if break_on_hyphens { + split_chunks(&characters) + } else { + split_whitespace_chunks(&characters) + }; chunks.reverse(); let mut lines: Vec = Vec::new(); while !chunks.is_empty() { @@ -59,7 +68,12 @@ pub(crate) fn wrap(text: &str, width: usize, (initial, subsequent): (&str, &str) } } if let Some(chunk) = chunks.pop_if(|chunk| chunk.len() as i64 > available) { - let (head, rest) = split_long_word(chunk, line.is_empty(), length, available); + let (head, rest) = split_long_word( + chunk, + (line.is_empty(), break_on_hyphens), + length, + available, + ); line.extend(head); chunks.extend(rest); } @@ -77,7 +91,7 @@ pub(crate) fn wrap(text: &str, width: usize, (initial, subsequent): (&str, &str) /// `_handle_long_word`: the part of a too-long chunk the line takes, and what stays queued. fn split_long_word( mut chunk: Vec, - line_is_empty: bool, + (line_is_empty, break_on_hyphens): (bool, bool), length: usize, width: i64, ) -> (Option>, Option>) { @@ -91,7 +105,8 @@ fn split_long_word( } let space_left: usize = space_left as usize; let mut end: usize = space_left.min(chunk.len()); - if chunk.len() > space_left + if break_on_hyphens + && chunk.len() > space_left && let Some(hyphen) = chunk[..space_left] .iter() .rposition(|character| *character == '-') @@ -136,6 +151,22 @@ fn split_chunks(text: &[char]) -> Vec> { chunks } +/// `TextWrapper.wordsep_simple_re.split`: whitespace runs and the words between them. +fn split_whitespace_chunks(text: &[char]) -> Vec> { + let mut chunks: Vec> = Vec::new(); + let mut start: usize = 0; + while start < text.len() { + let space: bool = is_space(text[start]); + let length: usize = text[start..] + .iter() + .take_while(|character| is_space(**character) == space) + .count(); + chunks.push(text[start..start + length].to_vec()); + start += length; + } + chunks +} + fn at(text: &[char], index: usize) -> Option { text.get(index).copied() } diff --git a/crates/sqlbuild-core/src/text/main/text_wrap.rs b/crates/sqlbuild-core/src/text/main/text_wrap.rs index b2e32cccee..9f72651762 100644 --- a/crates/sqlbuild-core/src/text/main/text_wrap.rs +++ b/crates/sqlbuild-core/src/text/main/text_wrap.rs @@ -1,9 +1,14 @@ -//! Python's `textwrap.wrap` with its default options. +//! Python's `textwrap.wrap`. use crate::text::_helpers::text_wrap::wrap; -/// `textwrap.wrap(text, width, initial_indent=initial, subsequent_indent=subsequent)`. +/// `textwrap.wrap(text, width, initial_indent=initial, subsequent_indent=subsequent, break_on_hyphens=...)`. #[must_use] -pub fn text_wrap(text: &str, width: usize, indents: (&str, &str)) -> Vec { - wrap(text, width, indents) +pub fn text_wrap( + text: &str, + width: usize, + indents: (&str, &str), + break_on_hyphens: bool, +) -> Vec { + wrap(text, width, indents, break_on_hyphens) } diff --git a/crates/sqlbuild-core/src/text/tests/test_text_wrap.rs b/crates/sqlbuild-core/src/text/tests/test_text_wrap.rs index 7aabeeb99a..5126658f43 100644 --- a/crates/sqlbuild-core/src/text/tests/test_text_wrap.rs +++ b/crates/sqlbuild-core/src/text/tests/test_text_wrap.rs @@ -33,7 +33,7 @@ fn given_prose_when_wrapping_then_lines_match_python_textwrap() { for test_case in test_cases { assert_eq!( - text_wrap(test_case.text, test_case.width, test_case.indents), + text_wrap(test_case.text, test_case.width, test_case.indents, true), test_case.expected_lines, "{}", test_case.description diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs index b1cfc308a8..1e4d77872b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs @@ -10,4 +10,5 @@ pub(crate) mod queries; pub(crate) mod reference_facts; pub(crate) mod relation_names; mod required_columns; +pub(crate) mod schema_request; pub(crate) mod sql_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs index 0deadf4ef8..c7620efbe8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs @@ -1,30 +1,38 @@ //! The external columns a model's query reads. use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::required_columns::errors::RequiredColumnsError; use sqlbuild_analysis::required_columns::main::resolve_required_columns::resolve_required_columns as resolve; use sqlbuild_analysis::required_columns::models::{ColumnRead, ReferenceInput}; use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::types::CompilerDetach; -/// Each `(resource type, resource, column)` the query reads, given its `(kind, name)` references. +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; + +/// CTE-aware column reads of `query_sql`, resolved to the external resources it references. #[pyfunction] -#[pyo3(signature = (query_sql, references, dialect))] -fn resolve_required_columns( - py: Python<'_>, +#[pyo3(signature = (*, query_sql, references, dialect))] +fn analyze_resolved_column_reads<'py>( + py: Python<'py>, query_sql: &str, - references: Vec<(String, String)>, + references: &Bound<'py, PyAny>, dialect: Option<&str>, -) -> PyResult> { - let references: Vec = references - .into_iter() - .map(|(kind, name)| ReferenceInput { kind, name }) - .collect(); +) -> PyResult> { + let mut inputs: Vec = Vec::new(); + for reference in references.try_iter()? { + let reference: Bound<'py, PyAny> = reference?; + inputs.push(ReferenceInput { + kind: reference.getattr("ref_kind")?.str()?.extract()?, + name: reference.getattr("ref_name")?.extract()?, + }); + } let reads: Vec = py - .compiler_detach(|| Ok(resolve(query_sql, &references, dialect))) + .compiler_detach(|| Ok(resolve(query_sql, &inputs, dialect))) .map_err(value_error)? .map_err(|error| match error { RequiredColumnsError::UnknownDialect(name) => { @@ -32,12 +40,30 @@ fn resolve_required_columns( } RequiredColumnsError::Normalization(message) => value_error(message), })?; - Ok(reads - .into_iter() - .map(|read| (read.resource_type, read.resource_name, read.column_name)) - .collect()) + let fact: Bound<'py, PyAny> = imported(py, COMPILE_MODELS_MODULE, "CompiledLineageSourceFact")?; + let mut facts: Vec> = Vec::with_capacity(reads.len()); + for read in reads { + facts.push(call_with( + &fact, + &[ + ( + "resource_type", + PyString::new(py, &read.resource_type).into_any(), + ), + ( + "resource_name", + PyString::new(py, &read.resource_name).into_any(), + ), + ( + "column_name", + PyString::new(py, &read.column_name).into_any(), + ), + ], + )?); + } + PyTuple::new(py, facts) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_required_columns, module)?) + module.add_function(wrap_pyfunction!(analyze_resolved_column_reads, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs new file mode 100644 index 0000000000..4d1ddd93fe --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs @@ -0,0 +1,66 @@ +//! Build the stable native schema-validation request of one model query. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_analysis::semantic_validation::main::normalize::normalize_analysis_sql; +use sqlbuild_analysis::semantic_validation::models::NormalizationInput; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::types::CompilerDetach; + +const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; +const GENERIC_DIALECT: &str = "generic"; + +/// The query, relation schemas and binding context of one schema-validation request. +pub(crate) struct SchemaRequestParts<'py> { + pub(crate) query_sql: String, + pub(crate) placeholders: Bound<'py, PyAny>, + pub(crate) dialect: Bound<'py, PyAny>, + pub(crate) schema: Bound<'py, PyAny>, + pub(crate) known_functions: Bound<'py, PyAny>, + pub(crate) known_types: Bound<'py, PyAny>, + pub(crate) catalog: Bound<'py, PyAny>, + pub(crate) quoted_identifiers_ignore_case: bool, +} + +/// A `SqlSchemaValidationRequest` over the normalized analysis SQL of `parts.query_sql`. +pub(crate) fn schema_validation_request<'py>( + py: Python<'py>, + parts: SchemaRequestParts<'py>, +) -> PyResult> { + let dialect: String = if parts.dialect.is_truthy()? { + parts.dialect.extract()? + } else { + GENERIC_DIALECT.to_owned() + }; + let placeholders: HashMap = if parts.placeholders.is_truthy()? { + parts.placeholders.extract()? + } else { + HashMap::new() + }; + let input: NormalizationInput = NormalizationInput { + sql: parts.query_sql, + dialect, + stubs: HashMap::new(), + placeholders, + }; + let sql: String = py + .compiler_detach(|| normalize_analysis_sql(input)) + .map_err(value_error)?; + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("sql", sql)?; + arguments.set_item("dialect", parts.dialect)?; + arguments.set_item("schema", parts.schema)?; + arguments.set_item("known_functions", parts.known_functions)?; + arguments.set_item("known_types", parts.known_types)?; + arguments.set_item( + "quoted_identifiers_ignore_case", + parts.quoted_identifiers_ignore_case, + )?; + arguments.set_item("catalog", parts.catalog)?; + py.import(SQL_ANALYSIS_MODELS_MODULE)? + .getattr("SqlSchemaValidationRequest")? + .call((), Some(&arguments)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs new file mode 100644 index 0000000000..28f000ad13 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs @@ -0,0 +1,88 @@ +//! Fixture-column inference: columns plus explicitly untyped NULL projections. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::analysis_session::main::projection_facts::projection_facts; +use sqlbuild_analysis::assembly::analysis_session::models::ProjectionFacts; +use sqlbuild_core::panics::main::native_failure::native_failure; +use sqlbuild_core::text::main::python_casefold::python_casefold; + +use crate::bindings::_helpers::analysis_session::query_columns::ColumnsRow; +use crate::bindings::_helpers::analysis_session::stage_functions::query_column_rows; +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::types::CompilerDetach; + +const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const PROJECTION_FACTS_CONTEXT: &str = "native fixture projection facts"; +const LEGACY_MODE: &str = "legacy"; + +/// `infer_fixture_column_facts(*, query_sql, inference_profile)`: a `FixtureColumnInference`. +#[pyfunction] +#[pyo3(signature = (**request))] +fn infer_fixture_column_facts<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("infer_fixture_column_facts", request)?; + let py: Python<'py> = request.py(); + let query_sql: String = request.get("query_sql")?.extract()?; + let profile: Bound<'py, PyAny> = request.get("inference_profile")?; + let query: Bound<'py, PyDict> = PyDict::new(py); + query.set_item("sql", &query_sql)?; + query.set_item("mode", LEGACY_MODE)?; + query.set_item("recover_cte_facts", false)?; + let query: Bound<'py, PyAny> = py + .import(ANALYSIS_MODELS_MODULE)? + .getattr("NativeColumnQuery")? + .call((), Some(&query))?; + let empty: Bound<'py, PyAny> = PyDict::new(py).into_any(); + let rows: Vec = query_column_rows( + PyTuple::new(py, [query])?.as_any(), + &profile, + (&empty, &empty), + )?; + let Some((true, Some(columns), false)) = rows.into_iter().next() else { + return Ok(py.None().into_bound(py)); + }; + let dialect: Option = profile.getattr("sql_analysis_dialect")?.extract()?; + let facts: ProjectionFacts = py + .compiler_detach(|| projection_facts(&query_sql, dialect.as_deref())) + .map_err(|reason| compiler_error(native_failure(PROJECTION_FACTS_CONTEXT, &reason)))?; + let simple_types: HashMap = facts.types.into_iter().collect(); + let column_class: Bound<'py, PyAny> = py + .import(COMPILE_MODELS_MODULE)? + .getattr("InferredColumn")?; + let nullability_class: Bound<'py, PyAny> = py + .import(LINEAGE_TYPES_MODULE)? + .getattr("InferredNullability")?; + let mut built: Vec> = Vec::with_capacity(columns.len()); + for (name, data_type, nullability) in columns { + let data_type: Option = data_type + .filter(|data_type| !data_type.is_empty()) + .or_else(|| simple_types.get(&python_casefold(&name)).cloned()); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("name", name)?; + arguments.set_item("type", data_type)?; + arguments.set_item("nullability", nullability_class.call1((nullability,))?)?; + built.push(column_class.call((), Some(&arguments))?); + } + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("columns", PyTuple::new(py, built)?)?; + arguments.set_item( + "null_literal_names", + PyFrozenSet::new(py, facts.null_literal_names)?, + )?; + arguments.set_item("quoted_names", PyFrozenSet::new(py, facts.quoted_names)?)?; + py.import(COMPILE_MODELS_MODULE)? + .getattr("FixtureColumnInference")? + .call((), Some(&arguments)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(infer_fixture_column_facts, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs index d212ed073a..baca587c27 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod fixture_columns; pub(crate) mod model_analyses; pub(crate) mod nullability_rules; pub(crate) mod oracle; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 5fb7a0b6f9..7dea30fd8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -9,13 +9,12 @@ use crate::bindings::_helpers::analysis_session::profile_inputs::{ case_sensitive_shapes, nullability_rules, profile_dialect, return_types, shape_rows, }; use crate::bindings::_helpers::analysis_session::stage_functions::family_rows; +use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::boundary::keyword_request::Request; const NATIVE_MODULE: &str = "sqlbuild._native"; const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; const ANALYSIS_CACHE_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.cache"; -const CURSOR_INTRINSICS_MODULE: &str = - "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const CODE_IDENTITY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiled_code_identity"; @@ -277,9 +276,6 @@ fn model_rows<'py>( py: Python<'py>, model_inputs: &[Bound<'py, PyAny>], ) -> PyResult> { - let cursor_sql: Bound<'py, PyAny> = py - .import(CURSOR_INTRINSICS_MODULE)? - .getattr("cursor_intrinsics_analysis_sql")?; let planner = py.import(PLANNER_TYPES_MODULE)?; let enforced: Bound<'py, PyAny> = planner.getattr("ContractPolicy")?.getattr("ENFORCED")?; let snapshot: Bound<'py, PyAny> = planner @@ -289,10 +285,15 @@ fn model_rows<'py>( for model_input in model_inputs { let values: Bound<'py, PyAny> = model_input.getattr("config")?.getattr("values")?; let get = |key: &str| values.call_method1("get", (key,)); - let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("sql", model_input.getattr("query_sql")?)?; - arguments.set_item("cursor_type", get("cursor_type")?)?; - let query_sql: Bound<'py, PyAny> = cursor_sql.call((), Some(&arguments))?; + let query_sql: Bound<'py, PyAny> = PyString::new( + py, + &analysis_cursor_sql( + py, + &model_input.getattr("query_sql")?.extract::()?, + &get("cursor_type")?, + )?, + ) + .into_any(); let mut placeholders: Vec<(String, String)> = Vec::new(); let raw: Bound<'py, PyAny> = get("placeholders")?; if raw.is_instance_of::() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs index b0a64e9832..04b7a39550 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs @@ -3,11 +3,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::analysis_session::{ - model_analyses, oracle, outcome_objects, query_columns, session, stage_functions, + fixture_columns, model_analyses, oracle, outcome_objects, query_columns, session, + stage_functions, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { session::register(module)?; + fixture_columns::register(module)?; model_analyses::register(module)?; outcome_objects::register(module)?; query_columns::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs index 0414970f55..7843499028 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs @@ -243,7 +243,7 @@ fn or_empty<'py>( } } -fn query_column_rows( +pub(crate) fn query_column_rows( queries: &Bound<'_, PyAny>, profile: &Bound<'_, PyAny>, (types, nullability): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs index b36dcc7f9d..e942a97baf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs @@ -1,76 +1,157 @@ //! Cursor intrinsic validation, rejection and rendering. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{PyString, PyStringMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::{Python, pyfunction, wrap_pyfunction}; use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; use sqlbuild_attachments::cursor_intrinsics::main::validated_model_intrinsics::validated_model_intrinsics; use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::compile_driver::python_calls::imported; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const INCREMENTAL_MATERIALIZATION: &str = "incremental"; const CURSOR_START: &str = "__cursor_start"; +const CURSOR_START_CALL: &str = "__cursor_start()"; +const CURSOR_END_CALL: &str = "__cursor_end()"; +const INTEGER_CURSOR_TYPE: &str = "integer"; +const INTEGER_ANALYSIS_BOUND: &str = "CAST(0 AS BIGINT)"; +const TIMESTAMP_ANALYSIS_BOUND: &str = "CAST('2000-01-01 00:00:00' AS TIMESTAMP)"; +const MODEL_SQL_CONTEXT: &str = "Model SQL"; +const SQL_CONTEXT: &str = "SQL"; +const RESERVED_CURSOR_MARKERS: [&str; 2] = ["__SQB_CURSOR_START__", "__SQB_CURSOR_END__"]; -/// A model query with canonical intrinsic calls and no error, or no query and the error. -#[pyfunction] -fn validated_model_cursor_intrinsics( - sql: &str, - reserved_markers: Vec, - model: (String, Bound<'_, PyAny>, Bound<'_, PyAny>), -) -> PyResult<(Option, Option)> { - let (name, materialized, cursor) = model; - let incremental: bool = materialized - .downcast::() - .is_ok_and(|text| text.to_string_lossy() == INCREMENTAL_MATERIALIZATION); - let cursor: Option = match cursor.downcast::() { +/// The reserved executor bound markers no authored SQL may contain. +pub(crate) fn reserved_cursor_markers() -> Vec { + RESERVED_CURSOR_MARKERS + .iter() + .map(|marker| (*marker).to_owned()) + .collect() +} + +/// A `CompileInputError` carrying `message`. +fn compile_input_error(py: Python<'_>, message: String) -> PyResult { + Ok(PyErr::from_value( + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?.call1((message,))?, + )) +} + +/// `text` when it is a string, else `None`. +fn text_value(value: &Bound<'_, PyAny>) -> Option { + match value.downcast::() { Ok(text) => Some(text.to_string_lossy().into_owned()), Err(_) => None, - }; - compiler_guard(|| { - Ok( - match validated_model_intrinsics( - sql, - &reserved_markers, - &IntrinsicModel { - name: &name, - incremental, - cursor: cursor.as_deref(), - }, - ) { - Ok(canonical) => (Some(canonical), None), - Err(message) => (None, Some(message)), - }, - ) - }) + } } -/// `sql` with intrinsic calls replaced and whether any was, or the error for `context`. -#[pyfunction] -fn replace_cursor_intrinsics( +/// `sql` with each intrinsic replaced by its bound and whether any was, raising for `context`. +fn replaced( + py: Python<'_>, sql: &str, context: &str, - replacements: (String, String), -) -> PyResult<(String, bool, Option)> { - let (start, end) = replacements; - compiler_guard(|| { - Ok( - match replace_intrinsics(sql, context, |name| { - if name == CURSOR_START { - start.clone() - } else { - end.clone() - } - }) { - Ok((replaced, found)) => (replaced, found, None), - Err(message) => (String::new(), false, Some(message)), + (start, end): (&str, &str), +) -> PyResult<(String, bool)> { + let outcome: Result<(String, bool), String> = compiler_guard(|| { + Ok(replace_intrinsics(sql, context, |name| { + if name == CURSOR_START { + start.to_owned() + } else { + end.to_owned() + } + })) + })?; + outcome.or_else(|message| Err(compile_input_error(py, message)?)) +} + +/// One model query with canonical intrinsic calls, raising when the model may not use them. +pub(crate) fn validated_model_cursor_sql( + py: Python<'_>, + sql: &str, + (model_name, config_values): (&str, &Bound<'_, PyAny>), +) -> PyResult { + let incremental: bool = text_value(&config_values.call_method1("get", ("materialized",))?) + .is_some_and(|materialized| materialized == INCREMENTAL_MATERIALIZATION); + let cursor: Option = text_value(&config_values.call_method1("get", ("cursor",))?); + let reserved_markers: Vec = reserved_cursor_markers(); + let outcome: Result = compiler_guard(|| { + Ok(validated_model_intrinsics( + sql, + &reserved_markers, + &IntrinsicModel { + name: model_name, + incremental, + cursor: cursor.as_deref(), }, - ) - }) + )) + })?; + outcome.or_else(|message| Err(compile_input_error(py, message)?)) +} + +/// `sql` with intrinsics replaced by stable typed literals, only for static SQL analysis. +pub(crate) fn analysis_cursor_sql( + py: Python<'_>, + sql: &str, + cursor_type: &Bound<'_, PyAny>, +) -> PyResult { + let literal: &str = if text_value(cursor_type).as_deref() == Some(INTEGER_CURSOR_TYPE) { + INTEGER_ANALYSIS_BOUND + } else { + TIMESTAMP_ANALYSIS_BOUND + }; + Ok(replaced(py, sql, MODEL_SQL_CONTEXT, (literal, literal))?.0) +} + +/// One model query with canonical intrinsic calls, raising `CompileInputError` when invalid. +#[pyfunction] +#[pyo3(signature = (*, sql, config_values, model_name))] +fn validated_model_cursor_intrinsics( + py: Python<'_>, + sql: &str, + config_values: &Bound<'_, PyAny>, + model_name: &str, +) -> PyResult { + validated_model_cursor_sql(py, sql, (model_name, config_values)) +} + +/// `sql` with intrinsics replaced by typed literals of `cursor_type` for static analysis. +#[pyfunction] +#[pyo3(signature = (*, sql, cursor_type))] +fn cursor_intrinsics_analysis_sql( + py: Python<'_>, + sql: &str, + cursor_type: &Bound<'_, PyAny>, +) -> PyResult { + analysis_cursor_sql(py, sql, cursor_type) +} + +/// Whether `sql` uses cursor intrinsics, and `sql` with both bounds rendered when given. +#[pyfunction] +#[pyo3(signature = (*, sql, start_sql=None, end_sql=None))] +fn resolve_cursor_intrinsics( + py: Python<'_>, + sql: &str, + start_sql: Option<&str>, + end_sql: Option<&str>, +) -> PyResult<(String, bool)> { + let (rendered, found) = replaced( + py, + sql, + SQL_CONTEXT, + ( + start_sql.unwrap_or(CURSOR_START_CALL), + end_sql.unwrap_or(CURSOR_END_CALL), + ), + )?; + if !found || start_sql.is_none() || end_sql.is_none() { + return Ok((sql.to_owned(), found)); + } + Ok((rendered, true)) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(validated_model_cursor_intrinsics, module)?)?; - module.add_function(wrap_pyfunction!(replace_cursor_intrinsics, module)?)?; + module.add_function(wrap_pyfunction!(cursor_intrinsics_analysis_sql, module)?)?; + module.add_function(wrap_pyfunction!(resolve_cursor_intrinsics, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index 4f9d85d50d..a438a7c984 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -17,10 +17,9 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; -const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main._static_command"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const AUDIT_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._audit"; -const PROGRESS_REPORTER_MODULE: &str = "sqlbuild.cli.progress.classes.audit_progress_reporter"; const COST_CONTEXT_MODULE: &str = "sqlbuild.cost.classes.cost_context"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const PROJECTION_MODULE: &str = "sqlbuild.executor.auditing.main._current_audit_result_projection"; @@ -196,7 +195,7 @@ fn prepare_run<'py>( let stream: Bound<'py, PyAny> = invocation.getattr("progress_stream")?; let use_color: Bound<'py, PyAny> = invocation.getattr("use_color")?; let progress: Bound<'py, PyAny> = call_with( - &imported(py, PROGRESS_REPORTER_MODULE, "AuditProgressReporter")?, + &imported(py, NATIVE_MODULE, "AuditProgressReporter")?, &[ ("entries", entries.clone()), ("worker_limit", PyInt::new(py, worker_count).into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs index 282e72911a..8e07eaa020 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs @@ -1,3 +1,3 @@ mod fields; -mod project_vars; +pub(crate) mod project_vars; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs index cd6f3e1c86..9ecb1ae789 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs @@ -49,6 +49,11 @@ fn preview_json(value: &Bound<'_, PyAny>) -> PyResult { #[pyfunction] #[pyo3(signature = (*, value, label))] fn render_project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { + project_var_text(value, label) +} + +/// One project var value as interpolated text; structured values are rejected. +pub(crate) fn project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { let py: Python<'_> = value.py(); if value.is_none() { return Ok(String::new()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index a4ef5802f0..7dd3ddb644 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -25,7 +25,6 @@ const COST_COLLECTION_MODULE: &str = "sqlbuild.cli.commands._helpers.cost.collec const COST_TYPES: &str = "sqlbuild.cost.types"; const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const BUILD_TYPES: &str = "sqlbuild.executor.build.types"; -const CALLBACKS_MODULE: &str = "sqlbuild.cli.commands.classes.build_progress_callbacks"; const RUNTIME_TARGET_MODULE: &str = "sqlbuild.cli.target_artifacts.main._write_runtime_target"; const CHECK_TARGET_MODULE: &str = "sqlbuild.cli.target_artifacts.main._write_python_check_runtime_target"; @@ -360,7 +359,7 @@ pub(crate) fn complete_direct_build<'py>( let cost_seconds: f64 = finish_cost_timing(context, cost_started); let record: Bound<'py, PyAny> = record?; let footer: String = call_with( - &imported(py, CALLBACKS_MODULE, "format_build_footer")?, + &imported(py, NATIVE_MODULE, "format_build_footer")?, &[ ("result", result.clone()), ("elapsed", preparation.callbacks.getattr("elapsed")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index b31ca9c1cf..61bbfd3bce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -17,7 +17,6 @@ use crate::bindings::_helpers::build_command::support::{ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const CALLBACKS_MODULE: &str = "sqlbuild.cli.commands.classes.build_progress_callbacks"; const PARSING_MODULE: &str = "sqlbuild.cli.commands._helpers.input.parsing"; const CURSOR_OVERRIDES_MODULE: &str = "sqlbuild.compiler.planner.main.cursor_window._typed_cursor_overrides"; @@ -207,7 +206,7 @@ pub(crate) fn prepare_build_execution<'py>( let plan: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; let debug: bool = context.request_flag("debug")? || context.request_flag("json_output")?; let callbacks: Bound<'py, PyAny> = call_with( - &imported(py, CALLBACKS_MODULE, "BuildProgressCallbacks")?, + &imported(py, NATIVE_MODULE, "BuildProgressCallbacks")?, &[ ("plan", plan.clone()), ("use_color", boolean(py, context.use_color)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 7ae951885e..3d109f203a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -15,11 +15,11 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const REPORTERS_MODULE: &str = "sqlbuild.cli.progress.main._build_command_progress_reporters"; -const SETTINGS_MODULE: &str = "sqlbuild.compiler.compile.main.effective_settings"; +const SETTINGS_MODULE: &str = "sqlbuild._native"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; -const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; -const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; +const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; const DEFER_CONFLICT_CODE: &str = "C408"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index 07ddd6113d..ad9c071ce9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -18,7 +18,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; -const RUN_LIFECYCLE_MODULE: &str = "sqlbuild.compiler.python_nodes.main._run_lifecycle"; const PYTHON_NODE_MODELS: &str = "sqlbuild.compiler.python_nodes.models"; const LOADER_DAG_MODULE: &str = "sqlbuild.compiler.planner.main.selection._loader_dag"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; @@ -316,7 +315,7 @@ fn lifecycle_plan<'py>( ], )?; let plan: Bound<'py, PyAny> = call_with( - &imported(py, RUN_LIFECYCLE_MODULE, "build_python_sql_run_lifecycle")?, + &imported(py, NATIVE_MODULE, "build_python_sql_run_lifecycle")?, &[("selection", selection), ("python_graph", graph.clone())], )?; let source_type: Bound<'py, PyAny> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index 9bd9b0908c..02bcdfc731 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -17,7 +17,6 @@ const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.exte const VERBOSE_INSPECTION_MODULE: &str = "sqlbuild.cli.progress.main.verbose_inspection_output"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; -const MANIFEST_MODULE: &str = "sqlbuild.compiler.manifest.main.build"; const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; /// Clone out-of-selection boundaries from the defer-clone origin before planning. @@ -212,7 +211,7 @@ pub(crate) fn write_compile_target( let plan: Bound<'_, PyAny> = pipeline.getattr("plan_output")?; let manifest: Bound<'_, PyAny> = if context.request_flag("manifest")? { call_with( - &imported(py, MANIFEST_MODULE, "build_manifest")?, + &imported(py, NATIVE_MODULE, "build_manifest")?, &[ ("project", pipeline.getattr("project")?), ("plan_output", plan.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs new file mode 100644 index 0000000000..9724d0d04e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs @@ -0,0 +1,304 @@ +//! Aggregate live state and plan-ordered result rows of standalone concurrent audits. + +use std::collections::HashMap; +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::build_progress::models::AuditResultText; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::nested_progress::main::audit_group_name::audit_group_name; +use sqlbuild_core::nested_progress::main::audit_resource_id::audit_resource_id; +use sqlbuild_core::nested_progress::main::render_audit_aggregate::render_audit_aggregate; +use sqlbuild_core::nested_progress::main::render_audit_row::render_audit_row; +use sqlbuild_core::nested_progress::main::render_group_header::render_group_header; +use sqlbuild_core::nested_progress::models::AuditAggregate; + +use crate::bindings::_helpers::build_progress::result_text::{audit_text, optional, text}; +use crate::bindings::_helpers::build_progress::terminal::{ + CLEAR_LINE, HIDE_CURSOR, SHOW_CURSOR, claim, consume_terminal, flush, projector, release, + shared_lines, tty_flag, write, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, within}; + +/// The identity an audit plan entry or result reports to the progress projector. +fn resource_id(value: &Bound<'_, PyAny>, name_field: &str) -> PyResult { + let kind: Option = match optional(value, "attached_target_kind")? { + Some(kind) => Some(kind), + None => Some(text(value, "attachment_kind")?), + }; + Ok(audit_resource_id( + &text(value, name_field)?, + kind.as_deref().unwrap_or(""), + ( + optional(value, "attached_target_name")?.as_deref(), + optional(value, "attached_column_name")?.as_deref(), + ), + )) +} + +/// Mutable reporter state; only held briefly and never across Python calls. +#[derive(Default)] +struct ReporterState { + completed_by_index: HashMap>>, + next_index: usize, + running: usize, + completed: usize, + current_group: Option, + closed: bool, + cursor_hidden: bool, + on_result: Option>, +} + +/// Render aggregate live state and flush completed audit rows in plan order. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct AuditProgressReporter { + stream: Py, + lines: Py, + lock: Py, + style: CliStyle, + is_tty: bool, + projector: Option>, + indexes: HashMap, + total: usize, + worker_limit: i64, + state: Mutex, +} + +impl AuditProgressReporter { + fn state(&self) -> MutexGuard<'_, ReporterState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn index_of(&self, identity: String) -> PyResult { + self.indexes + .get(&identity) + .copied() + .ok_or_else(|| PyKeyError::new_err(identity)) + } + + fn render_aggregate(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + let (closed, cursor_hidden, running, completed) = { + let state = reporter.state(); + ( + state.closed, + state.cursor_hidden, + state.running, + state.completed, + ) + }; + if !reporter.is_tty || closed { + return Ok(()); + } + let stream: &Bound<'_, PyAny> = reporter.stream.bind(py); + claim(reporter.lines.bind(py), stream, slf.as_any())?; + if !cursor_hidden { + write(stream, HIDE_CURSOR)?; + reporter.state().cursor_hidden = true; + } + write( + stream, + &render_audit_aggregate(AuditAggregate { + running, + completed, + total: reporter.total, + workers: reporter.worker_limit, + }), + )?; + flush(stream) + } + + fn flush_ready(&self, py: Python<'_>) -> PyResult<()> { + loop { + let ready: Option> = { + let mut state = self.state(); + let index: usize = state.next_index; + let Some(result) = state.completed_by_index.remove(&index) else { + return Ok(()); + }; + state.next_index += 1; + result + }; + if let Some(result) = ready { + self.write_result(py, result.bind(py))?; + } + } + } + + fn write_result(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + if self.is_tty { + write(stream, CLEAR_LINE)?; + } + let audit: AuditResultText = audit_text(result)?; + let group: String = audit_group_name(&audit); + let previous: Option = self.state().current_group.clone(); + if previous.as_ref() != Some(&group) { + write( + stream, + &render_group_header(&group, previous.is_none(), self.style), + )?; + self.state().current_group = Some(group); + } + let duration_ms: Option = match &self.projector { + Some(projector) => consume_terminal( + projector.bind(py), + &audit.audit_name, + Some(&resource_id(result, "audit_name")?), + )?, + None => None, + }; + write(stream, &render_audit_row(&audit, duration_ms, self.style))?; + flush(stream)?; + let callback: Option> = self + .state() + .on_result + .as_ref() + .map(|callback| callback.clone_ref(py)); + if let Some(callback) = callback { + callback.bind(py).call1((result,))?; + } + Ok(()) + } + + fn locked(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { + within(self.lock.bind(py), |_| body()) + } +} + +#[pymethods] +impl AuditProgressReporter { + #[new] + #[pyo3(signature = (**options))] + fn new(py: Python<'_>, options: Option>) -> PyResult { + let arguments = KeywordArguments::read( + py, + "AuditProgressReporter", + options, + (&["entries", "worker_limit", "stream", "use_color"], &[]), + )?; + let stream: Bound<'_, PyAny> = arguments.value("stream")?; + let lines: Bound<'_, PyAny> = shared_lines(py)?; + let mut indexes: HashMap = HashMap::new(); + let mut total: usize = 0; + for entry in arguments.value("entries")?.try_iter()? { + indexes.insert(resource_id(&entry?, "name")?, total); + total += 1; + } + Ok(Self { + is_tty: tty_flag(&stream)?, + stream: stream.unbind(), + lock: lines.getattr("lock")?.unbind(), + lines: lines.unbind(), + style: CliStyle { + use_color: arguments.value("use_color")?.is_truthy()?, + }, + projector: projector(py)?, + indexes, + total, + worker_limit: arguments.value("worker_limit")?.extract()?, + state: Mutex::new(ReporterState::default()), + }) + } + + /// Install plan-ordered result enrichment before dispatch starts. + fn set_result_callback(&self, callback: Py) { + self.state().on_result = Some(callback); + } + + /// Record a worker start without mutating one-item spinner state. + fn on_item_start(slf: &Bound<'_, Self>, entry: &Bound<'_, PyAny>) -> PyResult<()> { + let _ = entry; + slf.get().locked(slf.py(), || { + slf.get().state().running += 1; + Self::render_aggregate(slf) + }) + } + + /// Flush every newly contiguous plan-order result row. + fn on_item_complete(slf: &Bound<'_, Self>, result: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + reporter.locked(py, || { + let index: usize = reporter.index_of(resource_id(result, "audit_name")?)?; + reporter + .state() + .completed_by_index + .insert(index, Some(result.clone().unbind())); + reporter.flush_ready(py)?; + Self::render_aggregate(slf) + }) + } + + /// Update aggregate state immediately without projecting a result row. + fn on_item_physical_complete(slf: &Bound<'_, Self>, result: &Bound<'_, PyAny>) -> PyResult<()> { + let _ = result; + slf.get().locked(slf.py(), || { + { + let mut state = slf.get().state(); + state.running = state.running.saturating_sub(1); + state.completed += 1; + } + Self::render_aggregate(slf) + }) + } + + /// Advance aggregate and ordering state for a fatal audit exception. + fn on_item_error(slf: &Bound<'_, Self>, entry: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + reporter.locked(py, || { + let index: usize = reporter.index_of(resource_id(entry, "name")?)?; + { + let mut state = reporter.state(); + state.running = state.running.saturating_sub(1); + state.completed += 1; + state.completed_by_index.insert(index, None); + } + reporter.flush_ready(py)?; + Self::render_aggregate(slf) + }) + } + + /// Restore the terminal after the final aggregate update. + fn close(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + reporter.locked(py, || { + reporter.state().closed = true; + release(reporter.lines.bind(py), slf.as_any())?; + if !reporter.is_tty { + return Ok(()); + } + let stream: &Bound<'_, PyAny> = reporter.stream.bind(py); + write(stream, CLEAR_LINE)?; + let hidden: bool = std::mem::take(&mut reporter.state().cursor_hidden); + if hidden { + write(stream, SHOW_CURSOR)?; + } + flush(stream) + }) + } + + /// Erase the aggregate row so a persistent line can take its place. + fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, CLEAR_LINE)?; + flush(stream) + }) + } + + /// Draw the aggregate row again below persistent output. + fn redraw_transient_line(slf: &Bound<'_, Self>) -> PyResult<()> { + slf.get().locked(slf.py(), || Self::render_aggregate(slf)) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs new file mode 100644 index 0000000000..472ad71d1d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs @@ -0,0 +1,614 @@ +//! Live progress rows, completion rows and runtime diagnostics of `sqb build`. + +use std::collections::HashMap; +use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::time::Instant; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::build_progress::main::query_diagnostic_message::query_diagnostic_message; +use sqlbuild_core::build_progress::main::render_build_spinner_line::render_build_spinner_line; +use sqlbuild_core::build_progress::main::render_failed_test_result::render_failed_test_result; +use sqlbuild_core::build_progress::main::render_model_result::render_model_result; +use sqlbuild_core::build_progress::main::render_resource_result::render_resource_result; +use sqlbuild_core::build_progress::main::runtime_diagnostic_summary::runtime_diagnostic_summary; +use sqlbuild_core::build_progress::main::scheduler_diagnostic_message::scheduler_diagnostic_message; +use sqlbuild_core::build_progress::models::{ + ModelRowFacts, ProgressLayout, ResourceResult, SchedulerStateText, SpinnerRow, StatementText, + TestResultText, +}; +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::bindings::_helpers::build_progress::plan_facts::{BuildPlanFacts, build_plan_facts}; +use crate::bindings::_helpers::build_progress::result_text::{ + function_text, load_text, model_text, optional, seed_text, test_text, text, +}; +use crate::bindings::_helpers::build_progress::terminal::{ + CLEAR_LINE, HIDE_CURSOR, SHOW_CURSOR, SPINNER_FRAMES, SPINNER_TICK_SECONDS, claim, + consume_terminal, flush, join_spinner, projector, release, shared_lines, start_spinner, + terminal_columns, tty_flag, write, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported, within}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const RUNTIME_LOGGER: &str = "sqlbuild.executor.runtime"; +const PASS_OUTCOME: &str = "pass"; +const SCHEDULER_DIAGNOSTIC_LIMIT: usize = 50; +const QUERY_DIAGNOSTIC_LIMIT: usize = 25; + +/// Which executor result type a completion reports. +#[derive(Clone, Copy, PartialEq, Eq)] +enum ResultKind { + Model, + Seed, + Function, + Load, + PythonNode, + SqlTest, + Other, +} + +fn result_kind(result: &Bound<'_, PyAny>) -> PyResult { + for (field, kind) in [ + ("model_name", ResultKind::Model), + ("seed_name", ResultKind::Seed), + ("function_name", ResultKind::Function), + ("loader_name", ResultKind::Load), + ("node_name", ResultKind::PythonNode), + ("test_name", ResultKind::SqlTest), + ] { + if result.hasattr(field)? { + return Ok(kind); + } + } + Ok(ResultKind::Other) +} + +fn sql_test_resource_id(result: &Bound<'_, PyAny>) -> PyResult { + let name: String = text(result, "test_name")?; + let path: Bound<'_, PyAny> = result.getattr("source_path")?; + let block: Bound<'_, PyAny> = result.getattr("block_index")?; + let case: Bound<'_, PyAny> = result.getattr("case_name")?; + if path.is_none() || block.is_none() || case.is_none() { + return Ok(format!("sql_test:{name}")); + } + Ok(format!( + "sql_test:{}:{}:{}", + path.call_method0("as_posix")?.str()?, + block.str()?, + case.str()? + )) +} + +/// The resource name and projector identity of a completion. +fn resource_identity( + result: &Bound<'_, PyAny>, + kind: ResultKind, +) -> PyResult> { + Ok(match kind { + ResultKind::Model => { + let name: String = text(result, "model_name")?; + Some((name.clone(), format!("model:{name}"))) + } + ResultKind::Seed => { + let name: String = text(result, "seed_name")?; + Some((name.clone(), format!("seed:{name}"))) + } + ResultKind::Function => { + let name: String = text(result, "function_name")?; + let kind: String = text(result, "function_kind")?; + Some((name.clone(), format!("{kind}:{name}"))) + } + ResultKind::Load => { + let name: String = text(result, "source_name")?; + Some((name.clone(), format!("source:{name}"))) + } + ResultKind::PythonNode => { + let name: String = text(result, "node_name")?; + let kind: String = text(result, "kind")?; + Some((name.clone(), format!("{kind}:{name}"))) + } + ResultKind::SqlTest => Some((text(result, "test_name")?, sql_test_resource_id(result)?)), + ResultKind::Other => None, + }) +} + +/// Mutable callback state; only held briefly and never across Python calls. +#[derive(Default)] +struct CallbackState { + counter: usize, + node_name: String, + node_kind: String, + sub_message: String, + frame: usize, + line_active: bool, + cursor_hidden: bool, + stop_event: Option>, + thread: Option>, + tests_by_model: HashMap>, + scheduler_count: usize, + scheduler_omitted: usize, + last_scheduler: Option, + query_count: usize, + query_omitted: usize, + last_query: Option, +} + +/// Encapsulates live build progress output state and callbacks. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct BuildProgressCallbacks { + plan: Py, + stream: Py, + lines: Py, + lock: Py, + event_writer: Py, + projector: Option>, + facts: BuildPlanFacts, + style: CliStyle, + verbose: bool, + debug: bool, + is_tty: bool, + started: Instant, + state: Mutex, +} + +impl BuildProgressCallbacks { + fn state(&self) -> MutexGuard<'_, CallbackState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn locked(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { + within(self.lock.bind(py), |_| body()) + } + + fn layout(&self, py: Python<'_>) -> PyResult { + Ok(ProgressLayout { + total: self.facts.total, + name_width: self.facts.name_width, + columns: terminal_columns(py)?, + verbose: self.verbose, + }) + } + + fn emit(&self, py: Python<'_>, text: &str) -> PyResult<()> { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, text)?; + flush(stream) + } + + fn write_spinner_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || { + let (row, frame) = { + let mut state = self.state(); + if !state.line_active { + return Ok(()); + } + let frame: usize = state.frame; + state.frame = (frame + 1) % SPINNER_FRAMES.len(); + ( + SpinnerRow { + position: state.counter + 1, + resource_kind: state.node_kind.clone(), + name: state.node_name.clone(), + message: state.sub_message.clone(), + }, + frame, + ) + }; + self.emit( + py, + &render_build_spinner_line(&row, frame, self.layout(py)?, self.style), + ) + }) + } + + fn set_cursor(&self, py: Python<'_>, hidden: bool) -> PyResult<()> { + if self.state().cursor_hidden == hidden { + return Ok(()); + } + self.locked(py, || { + self.emit(py, if hidden { HIDE_CURSOR } else { SHOW_CURSOR }) + })?; + self.state().cursor_hidden = hidden; + Ok(()) + } + + fn stop_spinner(&self, py: Python<'_>) -> PyResult<()> { + let (event, thread) = { + let mut state = self.state(); + (state.stop_event.take(), state.thread.take()) + }; + if let Some(event) = event { + event.bind(py).call_method0("set")?; + } + if let Some(thread) = thread { + join_spinner(thread.bind(py))?; + } + Ok(()) + } + + /// Stop the spinner and erase its row before a persistent row is written. + fn retire_live_row(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let callbacks: &Self = slf.get(); + callbacks.stop_spinner(py)?; + if !callbacks.is_tty { + return Ok(()); + } + callbacks.locked(py, || { + callbacks.state().line_active = false; + release(callbacks.lines.bind(py), slf.as_any())?; + callbacks.emit(py, CLEAR_LINE) + })?; + callbacks.set_cursor(py, false) + } + + fn write_runtime_diagnostic(&self, py: Python<'_>, message: &str) -> PyResult<()> { + imported(py, "logging", "getLogger")? + .call1((RUNTIME_LOGGER,))? + .call_method1("debug", (message,))?; + if self.debug { + return Ok(()); + } + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("stream", self.stream.bind(py))?; + options.set_item("text", format!("{message}\n"))?; + self.lines + .bind(py) + .call_method("write_persistent", (), Some(&options))?; + Ok(()) + } + + fn write_node_result( + slf: &Bound<'_, Self>, + result: &Bound<'_, PyAny>, + kind: ResultKind, + ) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let callbacks: &Self = slf.get(); + if kind == ResultKind::SqlTest { + let test: TestResultText = test_text(result)?; + if test.outcome != PASS_OUTCOME { + Self::retire_live_row(slf)?; + return callbacks.emit( + py, + &render_failed_test_result(&test, callbacks.layout(py)?, callbacks.style), + ); + } + let mut state = callbacks.state(); + for step in &test.steps { + state + .tests_by_model + .entry(step.model_name.clone()) + .or_default() + .push(test.clone()); + } + return Ok(()); + } + Self::retire_live_row(slf)?; + let counter: usize = { + let mut state = callbacks.state(); + state.counter += 1; + state.counter + }; + let layout: ProgressLayout = callbacks.layout(py)?; + let style: CliStyle = callbacks.style; + let text: String = match kind { + ResultKind::Seed => render_resource_result( + ResourceResult::Node(&seed_text(result)?), + counter, + layout, + style, + ), + ResultKind::Function => render_resource_result( + ResourceResult::Node(&function_text(result)?), + counter, + layout, + style, + ), + ResultKind::Load => render_resource_result( + ResourceResult::Load(&load_text(result)?), + counter, + layout, + style, + ), + ResultKind::Model => { + let model = model_text(result)?; + let facts: &ModelRowFacts = callbacks + .facts + .models + .get(&model.name) + .unwrap_or(&callbacks.facts.unplanned_model); + let tests: Vec = callbacks + .state() + .tests_by_model + .get(&model.name) + .cloned() + .unwrap_or_default(); + render_model_result((&model, facts, &tests), counter, layout, style) + } + _ => return Ok(()), + }; + callbacks.emit(py, &text) + } + + fn write_event(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { + let writer: &Bound<'_, PyAny> = self.event_writer.bind(py); + let write_one = |item: &Bound<'_, PyAny>| -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("result", item)?; + options.set_item("plan", self.plan.bind(py))?; + writer.call_method("write_build_result", (), Some(&options))?; + Ok(()) + }; + write_one(result)?; + if result_kind(result)? == ResultKind::Model { + for audit in result.getattr("audit_results")?.try_iter()? { + write_one(&audit?)?; + } + } + Ok(()) + } +} + +fn scheduler_state(state: &Bound<'_, PyAny>) -> PyResult { + Ok(SchedulerStateText { + running: state.getattr("running")?.extract()?, + ready: state.getattr("ready")?.extract()?, + waiting: state.getattr("waiting")?.extract()?, + limit: state.getattr("limit")?.extract()?, + aborted: state.getattr("aborted")?.extract()?, + }) +} + +fn statement(telemetry: &Bound<'_, PyAny>) -> PyResult { + Ok(StatementText { + query_id: optional(telemetry, "query_id")?, + status: text(telemetry, "status")?, + elapsed_seconds: telemetry.getattr("elapsed_seconds")?.extract()?, + resource_type: text(telemetry, "resource_type")?, + resource_name: text(telemetry, "resource_name")?, + phase: text(telemetry, "phase")?, + }) +} + +#[pymethods] +impl BuildProgressCallbacks { + #[new] + #[pyo3(signature = (**options))] + fn new(py: Python<'_>, options: Option>) -> PyResult { + let arguments = KeywordArguments::read( + py, + "BuildProgressCallbacks", + options, + ( + &["plan", "use_color"], + &["verbose", "debug", "event_output_path"], + ), + )?; + let flag = |name: &str| -> PyResult { + Ok(arguments.passed(name)? && arguments.value(name)?.is_truthy()?) + }; + let plan: Bound<'_, PyAny> = arguments.value("plan")?; + let debug: bool = flag("debug")?; + let sys: Bound<'_, PyAny> = py.import("sys")?.into_any(); + let stdout: Bound<'_, PyAny> = sys.getattr("stdout")?; + let stream: Bound<'_, PyAny> = if debug { + sys.getattr("stderr")? + } else { + stdout.clone() + }; + let path: Bound<'_, PyAny> = if arguments.passed("event_output_path")? { + arguments.value("event_output_path")? + } else { + py.None().into_bound(py) + }; + let writer_options: Bound<'_, PyDict> = PyDict::new(py); + writer_options.set_item("path", path)?; + let event_writer: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "ExecutionEventWriter")?.call((), Some(&writer_options))?; + let facts: BuildPlanFacts = build_plan_facts(&plan)?; + let projector: Option> = projector(py)?; + if let Some(projector) = &projector { + let projector: &Bound<'_, PyAny> = projector.bind(py); + let ordinals: Bound<'_, PyDict> = PyDict::new(py); + for (name, ordinal) in &facts.ordinals { + ordinals.set_item(name, ordinal)?; + } + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("ordinals", ordinals)?; + options.set_item("total", facts.total)?; + projector.call_method("configure_resources", (), Some(&options))?; + for name in &facts.test_names { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("resource_name", name)?; + projector.call_method("expect_resource_enrichment", (), Some(&options))?; + } + } + let lines: Bound<'_, PyAny> = shared_lines(py)?; + Ok(Self { + plan: plan.unbind(), + is_tty: tty_flag(&stdout)? && !debug, + stream: stream.unbind(), + lock: lines.getattr("lock")?.unbind(), + lines: lines.unbind(), + event_writer: event_writer.unbind(), + projector, + facts, + style: CliStyle { + use_color: arguments.value("use_color")?.is_truthy()?, + }, + verbose: flag("verbose")?, + debug, + started: Instant::now(), + state: Mutex::new(CallbackState::default()), + }) + } + + #[getter] + fn elapsed(&self) -> f64 { + self.started.elapsed().as_secs_f64() + } + + #[pyo3(signature = (*, name, resource_kind))] + fn on_node_start( + slf: &Bound<'_, Self>, + name: String, + resource_kind: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let callbacks: &Self = slf.get(); + if let Some(projector) = &callbacks.projector { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("resource_name", name)?; + projector + .bind(py) + .call_method("expect_resource_enrichment", (), Some(&options))?; + return Ok(()); + } + { + let mut state = callbacks.state(); + state.node_name = name; + state.node_kind = resource_kind.getattr("value")?.extract()?; + state.sub_message = String::new(); + } + if !callbacks.is_tty { + return Ok(()); + } + callbacks.set_cursor(py, true)?; + callbacks.locked(py, || { + callbacks.state().line_active = true; + claim( + callbacks.lines.bind(py), + callbacks.stream.bind(py), + slf.as_any(), + )?; + callbacks.write_spinner_line(py) + })?; + callbacks.stop_spinner(py)?; + let (event, thread) = start_spinner(slf.as_any())?; + let mut state = callbacks.state(); + state.stop_event = Some(event.unbind()); + state.thread = Some(thread.unbind()); + Ok(()) + } + + fn on_sub_progress(&self, py: Python<'_>, message: String) -> PyResult<()> { + self.state().sub_message = message; + if self.is_tty { + self.write_spinner_line(py)?; + } + Ok(()) + } + + /// Report one deduplicated concurrent scheduler state transition. + fn on_scheduler_state(&self, py: Python<'_>, state: &Bound<'_, PyAny>) -> PyResult<()> { + let state: SchedulerStateText = scheduler_state(state)?; + { + let mut callback_state = self.state(); + callback_state.scheduler_count += 1; + if callback_state.scheduler_count > SCHEDULER_DIAGNOSTIC_LIMIT { + callback_state.scheduler_omitted += 1; + callback_state.last_scheduler = Some(state); + return Ok(()); + } + } + self.write_runtime_diagnostic(py, &scheduler_diagnostic_message(state)) + } + + /// Report a bounded sample of completed warehouse statement identities. + fn on_statement_complete(&self, py: Python<'_>, telemetry: &Bound<'_, PyAny>) -> PyResult<()> { + let telemetry: StatementText = statement(telemetry)?; + { + let mut state = self.state(); + state.query_count += 1; + if state.query_count > QUERY_DIAGNOSTIC_LIMIT { + state.query_omitted += 1; + state.last_query = Some(telemetry); + return Ok(()); + } + } + self.write_runtime_diagnostic(py, &query_diagnostic_message(&telemetry)) + } + + /// Erase the live spinner row so a persistent line can take its place. + fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || self.emit(py, CLEAR_LINE)) + } + + /// Draw the live spinner row again below persistent output. + fn redraw_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.write_spinner_line(py) + } + + fn on_node_complete(slf: &Bound<'_, Self>, node_result: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let callbacks: &Self = slf.get(); + let kind: ResultKind = result_kind(node_result)?; + let mut result: Bound<'_, PyAny> = node_result.clone(); + if let Some(projector) = &callbacks.projector + && let Some((name, identity)) = resource_identity(node_result, kind)? + { + let Some(duration_ms) = consume_terminal(projector.bind(py), &name, Some(&identity))? + else { + return Ok(()); + }; + if matches!( + kind, + ResultKind::Model | ResultKind::Seed | ResultKind::Function | ResultKind::Load + ) { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("duration_ms", duration_ms.trunc() as i64)?; + result = + imported(py, "dataclasses", "replace")?.call((node_result,), Some(&options))?; + } + } + let written: PyResult<()> = Self::write_node_result(slf, &result, kind); + let evented: PyResult<()> = callbacks.write_event(py, &result); + written.and(evented) + } + + /// Write one result to the shared structured event channel. + fn write_execution_event(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { + self.write_event(py, result) + } + + /// Close the optional structured execution event stream. + fn close(&self, py: Python<'_>) -> PyResult<()> { + let (scheduler, query) = { + let state = self.state(); + ( + state + .last_scheduler + .map(|last| (state.scheduler_omitted, last)), + state + .last_query + .clone() + .map(|last| (state.query_omitted, last)), + ) + }; + for line in runtime_diagnostic_summary( + scheduler, + query.as_ref().map(|(omitted, last)| (*omitted, last)), + ) { + self.write_runtime_diagnostic(py, &line)?; + } + self.event_writer.bind(py).call_method0("close")?; + Ok(()) + } + + fn _spin_until_stopped(&self, py: Python<'_>, stop_event: &Bound<'_, PyAny>) -> PyResult<()> { + while !stop_event + .call_method1("wait", (SPINNER_TICK_SECONDS,))? + .is_truthy()? + { + self.write_spinner_line(py)?; + } + Ok(()) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs new file mode 100644 index 0000000000..15b9afca2f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs @@ -0,0 +1,36 @@ +//! The `sqb build` completion footer. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::build_progress::main::format_build_footer::format_build_footer as footer; +use sqlbuild_core::build_progress::models::PythonNodeText; +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::bindings::_helpers::build_progress::result_text::{build_text, python_node_text}; + +/// Render the build completion line, warnings, failures and skipped rollup. +#[pyfunction] +#[pyo3(signature = (*, result, elapsed, use_color, python_node_results = None))] +fn format_build_footer( + result: &Bound<'_, PyAny>, + elapsed: f64, + use_color: bool, + python_node_results: Option<&Bound<'_, PyAny>>, +) -> PyResult { + let mut python: Vec = Vec::new(); + if let Some(results) = python_node_results { + for node in results.try_iter()? { + python.push(python_node_text(&node?)?); + } + } + Ok(footer( + &build_text(result)?, + &python, + elapsed, + CliStyle { use_color }, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_build_footer, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs new file mode 100644 index 0000000000..e0efa09893 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod audit_progress; +pub(crate) mod build_callbacks; +pub(crate) mod footer; +pub(crate) mod nested_progress; +pub(crate) mod plan_facts; +pub(crate) mod registration; +pub(crate) mod result_text; +pub(crate) mod terminal; +pub(crate) mod transient_status; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs new file mode 100644 index 0000000000..fc3fbb5177 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs @@ -0,0 +1,392 @@ +//! One live spinner row for all running items and grouped completed rows of `sqb test`. + +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::build_progress::models::ErrorText; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::nested_progress::main::render_group_header::render_group_header; +use sqlbuild_core::nested_progress::main::render_nested_item::render_nested_item; +use sqlbuild_core::nested_progress::main::render_nested_spinner_line::render_nested_spinner_line; +use sqlbuild_core::nested_progress::models::{NestedChildRow, NestedItem, NestedLayout}; + +use crate::bindings::_helpers::build_progress::result_text::text; +use crate::bindings::_helpers::build_progress::terminal::{ + CLEAR_LINE, HIDE_CURSOR, SHOW_CURSOR, SPINNER_FRAMES, SPINNER_TICK_SECONDS, claim, + consume_terminal, flush, join_spinner, projector, release, shared_lines, start_spinner, + terminal_columns, tty_flag, write, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, within}; + +const MIN_NAME_WIDTH: usize = 50; +const ITEM_OPTIONS: (&[&str], &[&str]) = ( + &["group_name", "item_name"], + &[ + "status_text", + "detail", + "error_code", + "error_help", + "error_message", + "child_rows", + "canonical_resource_name", + "canonical_resource_id", + ], +); + +/// Mutable reporter state; only held briefly and never across Python calls. +#[derive(Default)] +struct ReporterState { + printed_group: Option, + running: Vec<(String, String)>, + frame: usize, + line_active: bool, + cursor_hidden: bool, + stop_event: Option>, + thread: Option>, +} + +/// Render one live spinner row for all running items and grouped completed rows. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NestedCommandProgressCallbacks { + stream: Py, + lines: Py, + lock: Py, + label: String, + name_width: usize, + style: CliStyle, + is_tty: bool, + sequential: bool, + projector: Option>, + state: Mutex, +} + +fn optional_argument(arguments: &KeywordArguments<'_>, name: &str) -> PyResult> { + if !arguments.passed(name)? { + return Ok(None); + } + let value: Bound<'_, PyAny> = arguments.value(name)?; + if value.is_none() { + return Ok(None); + } + value.extract().map(Some) +} + +fn child_rows(arguments: &KeywordArguments<'_>) -> PyResult> { + if !arguments.passed("child_rows")? { + return Ok(Vec::new()); + } + let mut rows: Vec = Vec::new(); + for row in arguments.value("child_rows")?.try_iter()? { + let row: Bound<'_, PyAny> = row?; + rows.push(NestedChildRow { + label: text(&row, "label")?, + name: text(&row, "name")?, + status_text: text(&row, "status_text")?, + detail: text(&row, "detail")?, + }); + } + Ok(rows) +} + +impl NestedCommandProgressCallbacks { + fn state(&self) -> MutexGuard<'_, ReporterState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn locked(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { + within(self.lock.bind(py), |_| body()) + } + + fn layout(&self, py: Python<'_>) -> PyResult { + Ok(NestedLayout { + label: self.label.clone(), + name_width: self.name_width, + columns: terminal_columns(py)?, + }) + } + + fn draw_spinner_line(&self, py: Python<'_>) -> PyResult<()> { + let (running, frame) = { + let mut state = self.state(); + let frame: usize = state.frame; + state.frame = (frame + 1) % SPINNER_FRAMES.len(); + let running: Vec = state.running.iter().map(|(item, _)| item.clone()).collect(); + (running, frame) + }; + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write( + stream, + &render_nested_spinner_line( + &running, + SPINNER_FRAMES[frame], + &self.layout(py)?, + self.style, + ), + )?; + flush(stream) + } + + fn clear_live_row(&self, py: Python<'_>) -> PyResult<()> { + if self.is_tty && self.state().line_active { + write(self.stream.bind(py), CLEAR_LINE)?; + } + Ok(()) + } + + fn write_group_header(&self, py: Python<'_>, group: &str) -> PyResult<()> { + let previous: Option = self.state().printed_group.clone(); + if previous.as_deref() == Some(group) { + return Ok(()); + } + self.state().printed_group = Some(group.to_owned()); + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write( + stream, + &render_group_header(group, previous.is_none(), self.style), + )?; + flush(stream) + } + + fn set_cursor(&self, py: Python<'_>, hidden: bool) -> PyResult<()> { + if self.state().cursor_hidden == hidden { + return Ok(()); + } + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, if hidden { HIDE_CURSOR } else { SHOW_CURSOR })?; + flush(stream)?; + self.state().cursor_hidden = hidden; + Ok(()) + } + + /// Redraw the live row for remaining items, or retire it; return a thread to join. + fn refresh_live_row(slf: &Bound<'_, Self>) -> PyResult>> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + if !reporter.is_tty { + return Ok(None); + } + if !reporter.state().running.is_empty() { + reporter.draw_spinner_line(py)?; + return Ok(None); + } + reporter.state().line_active = false; + release(reporter.lines.bind(py), slf.as_any())?; + reporter.set_cursor(py, false)?; + let (event, thread) = { + let mut state = reporter.state(); + (state.stop_event.take(), state.thread.take()) + }; + if let Some(event) = event { + event.bind(py).call_method0("set")?; + } + Ok(thread) + } +} + +#[pymethods] +impl NestedCommandProgressCallbacks { + #[new] + #[pyo3(signature = (**options))] + fn new(py: Python<'_>, options: Option>) -> PyResult { + let arguments = KeywordArguments::read( + py, + "NestedCommandProgressCallbacks", + options, + ( + &["total", "label", "stream", "use_color"], + &["name_width", "concurrency"], + ), + )?; + let stream: Bound<'_, PyAny> = arguments.value("stream")?; + let lines: Bound<'_, PyAny> = shared_lines(py)?; + let name_width: usize = if arguments.passed("name_width")? { + arguments.value("name_width")?.extract()? + } else { + MIN_NAME_WIDTH + }; + let concurrency: i64 = if arguments.passed("concurrency")? { + arguments.value("concurrency")?.extract()? + } else { + 1 + }; + Ok(Self { + is_tty: tty_flag(&stream)?, + stream: stream.unbind(), + lock: lines.getattr("lock")?.unbind(), + lines: lines.unbind(), + label: arguments.value("label")?.extract()?, + name_width: name_width.max(MIN_NAME_WIDTH), + style: CliStyle { + use_color: arguments.value("use_color")?.is_truthy()?, + }, + sequential: concurrency <= 1, + projector: projector(py)?, + state: Mutex::new(ReporterState::default()), + }) + } + + #[pyo3(signature = (**options))] + fn on_item_start(slf: &Bound<'_, Self>, options: Option>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + let arguments = KeywordArguments::read( + py, + "on_item_start", + options, + (&["group_name", "item_name"], &["canonical_resource_name"]), + )?; + let group: String = arguments.value("group_name")?.extract()?; + let item: String = arguments.value("item_name")?.extract()?; + if let Some(projector) = &reporter.projector { + let canonical: Option = + optional_argument(&arguments, "canonical_resource_name")?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item( + "resource_name", + canonical + .filter(|name| !name.is_empty()) + .unwrap_or(item.clone()), + )?; + projector + .bind(py) + .call_method("expect_resource_enrichment", (), Some(&options))?; + } + reporter.locked(py, || { + if reporter.sequential && reporter.state().running.is_empty() { + reporter.clear_live_row(py)?; + reporter.write_group_header(py, &group)?; + } + { + let mut state = reporter.state(); + match state.running.iter_mut().find(|(name, _)| *name == item) { + Some(entry) => entry.1 = group, + None => state.running.push((item, group)), + } + } + if !reporter.is_tty { + return Ok(()); + } + reporter.set_cursor(py, true)?; + reporter.state().line_active = true; + claim( + reporter.lines.bind(py), + reporter.stream.bind(py), + slf.as_any(), + )?; + reporter.draw_spinner_line(py)?; + if reporter.state().thread.is_none() { + let (event, thread) = start_spinner(slf.as_any())?; + let mut state = reporter.state(); + state.stop_event = Some(event.unbind()); + state.thread = Some(thread.unbind()); + } + Ok(()) + }) + } + + #[pyo3(signature = (**options))] + fn on_item_complete(slf: &Bound<'_, Self>, options: Option>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + let arguments = KeywordArguments::read(py, "on_item_complete", options, ITEM_OPTIONS)?; + let group: String = arguments.value("group_name")?.extract()?; + let item_name: String = arguments.value("item_name")?.extract()?; + let mut detail: String = optional_argument(&arguments, "detail")?.unwrap_or_default(); + let duration_ms: Option = match &reporter.projector { + Some(projector) => { + let canonical: Option = + optional_argument(&arguments, "canonical_resource_name")?; + consume_terminal( + projector.bind(py), + &canonical + .filter(|name| !name.is_empty()) + .unwrap_or(item_name.clone()), + optional_argument(&arguments, "canonical_resource_id")?.as_deref(), + )? + } + None => None, + }; + let stopped: Option> = reporter.locked(py, || { + let item_group: String = { + let mut state = reporter.state(); + match state + .running + .iter() + .position(|(name, _)| *name == item_name) + { + Some(index) => state.running.remove(index).1, + None => group, + } + }; + if reporter.projector.is_none() || duration_ms.is_some() { + if let Some(duration_ms) = duration_ms { + detail = format!("{detail} {:.2}s", duration_ms / 1000.0); + } + let item: NestedItem = NestedItem { + item_name, + status_text: optional_argument(&arguments, "status_text")?.unwrap_or_default(), + detail, + error: ErrorText { + code: optional_argument(&arguments, "error_code")?, + message: optional_argument(&arguments, "error_message")?, + help: optional_argument(&arguments, "error_help")?, + }, + child_rows: child_rows(&arguments)?, + }; + reporter.clear_live_row(py)?; + reporter.write_group_header(py, &item_group)?; + write( + reporter.stream.bind(py), + &render_nested_item(&item, &reporter.layout(py)?, reporter.style), + )?; + } + let stopped: Option> = Self::refresh_live_row(slf)?; + flush(reporter.stream.bind(py))?; + Ok(stopped) + })?; + if let Some(thread) = stopped { + join_spinner(thread.bind(py))?; + } + Ok(()) + } + + /// Erase the live spinner row so a persistent line can take its place. + fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, CLEAR_LINE)?; + flush(stream) + }) + } + + /// Draw the live spinner row again below persistent output. + fn redraw_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || { + if self.state().line_active { + self.draw_spinner_line(py)?; + } + Ok(()) + }) + } + + fn _spin_until_stopped(&self, py: Python<'_>, stop_event: &Bound<'_, PyAny>) -> PyResult<()> { + while !stop_event + .call_method1("wait", (SPINNER_TICK_SECONDS,))? + .is_truthy()? + { + self.locked(py, || { + if !stop_event.call_method0("is_set")?.is_truthy()? && self.state().line_active { + self.draw_spinner_line(py)?; + } + Ok(()) + })?; + } + Ok(()) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs new file mode 100644 index 0000000000..3c788606b1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs @@ -0,0 +1,120 @@ +//! The plan facts build progress rows need: positions, name width and model row labels. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_core::build_progress::main::progress_name_width::progress_name_width; +use sqlbuild_core::build_progress::models::{ModelRowFacts, NameWidthInputs}; +use sqlbuild_core::plan_display::main::model_execution_annotation::model_execution_annotation; +use sqlbuild_core::plan_display::main::model_resource_type::model_resource_type; +use sqlbuild_core::plan_display::models::ModelLabelFacts; + +use crate::bindings::_helpers::build_progress::result_text::text; +use crate::bindings::_helpers::plan_output::display::optional_facts; + +const SOURCE_TYPE: &str = "source"; +const SQL_TEST_TYPE: &str = "sql_test"; + +/// What a build run's progress rows read from its plan. +pub(crate) struct BuildPlanFacts { + pub(crate) total: usize, + /// Display positions of every progress row, for the progress projector. + pub(crate) ordinals: Vec<(String, usize)>, + pub(crate) test_names: Vec, + pub(crate) name_width: usize, + pub(crate) models: HashMap, + /// The labels of a result whose model is not in the plan. + pub(crate) unplanned_model: ModelRowFacts, +} + +fn row_facts(facts: Option<&ModelLabelFacts>, logical_ddl: Option) -> ModelRowFacts { + ModelRowFacts { + display_type: model_resource_type(facts).to_owned(), + annotation: model_execution_annotation(facts), + logical_ddl, + } +} + +fn names(plan: &Bound<'_, PyAny>, field: &str) -> PyResult> { + plan.getattr(field)? + .try_iter()? + .map(|entry| text(&entry?, "name")) + .collect() +} + +/// Whether the execution-order key is a source with a loader. +fn loaded_source(plan: &Bound<'_, PyAny>, name: &Bound<'_, PyAny>) -> PyResult { + let source: Bound<'_, PyAny> = plan.getattr("source_map")?.call_method1("get", (name,))?; + Ok(!source.is_none() && !source.getattr("loader")?.is_none()) +} + +fn test_tree_children(plan: &Bound<'_, PyAny>) -> PyResult> { + let mut children: Vec = Vec::new(); + for entry in plan.getattr("test_entries")?.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + for step in entry.getattr("chain")?.try_iter()? { + let step: Bound<'_, PyAny> = step?; + if step.getattr("expected_cte_sql")?.is_truthy()? { + children.push(text(&step, "model_name")?); + } + } + for assertion in entry.getattr("assertions")?.try_iter()? { + children.push(format!("assertion {}", text(&assertion?, "name")?)); + } + } + Ok(children) +} + +pub(crate) fn build_plan_facts(plan: &Bound<'_, PyAny>) -> PyResult { + let mut models: HashMap = HashMap::new(); + let mut row_names: Vec = Vec::new(); + for entry in plan.getattr("model_entries")?.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + let name: String = text(&entry, "name")?; + let facts: ModelRowFacts = row_facts( + optional_facts(&entry)?.as_ref(), + Some(text(&entry, "logical_ddl")?), + ); + row_names.push(if facts.annotation.is_empty() { + name.clone() + } else { + format!("{name} ({})", facts.annotation) + }); + models.insert(name, facts); + } + let model_count: usize = models.len(); + let seeds: Vec = names(plan, "seed_entries")?; + let functions: Vec = names(plan, "function_entries")?; + let test_names: Vec = names(plan, "test_entries")?; + let mut ordinals: Vec<(String, usize)> = Vec::new(); + let mut loaded_sources: usize = 0; + for key in plan.getattr("execution_order")?.try_iter()? { + let key: Bound<'_, PyAny> = key?; + let resource_type: String = text(&key, "resource_type")?; + let name: Bound<'_, PyAny> = key.getattr("name")?; + if resource_type == SQL_TEST_TYPE { + continue; + } + if resource_type == SOURCE_TYPE { + if !loaded_source(plan, &name)? { + continue; + } + loaded_sources += 1; + } + ordinals.push((name.str()?.to_string(), ordinals.len() + 1)); + } + row_names.extend(seeds.iter().cloned()); + row_names.extend(functions.iter().cloned()); + row_names.extend(test_names.iter().cloned()); + Ok(BuildPlanFacts { + total: model_count + seeds.len() + functions.len() + loaded_sources, + ordinals, + name_width: progress_name_width(&NameWidthInputs { + names: row_names, + tree_child_names: test_tree_children(plan)?, + }), + test_names, + models, + unplanned_model: row_facts(None, None), + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs new file mode 100644 index 0000000000..591e3c48e3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs @@ -0,0 +1,15 @@ +//! Register the native build, audit and test progress reporters, the build footer and the transient status. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::build_progress::{ + audit_progress, build_callbacks, footer, nested_progress, transient_status, +}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + audit_progress::register(module)?; + build_callbacks::register(module)?; + footer::register(module)?; + nested_progress::register(module)?; + transient_status::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs new file mode 100644 index 0000000000..d02c1b6bad --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs @@ -0,0 +1,243 @@ +//! Executor result objects read into the plain data build progress output renders. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_core::build_progress::models::{ + AuditResultText, BuildResultText, ErrorText, HookResultText, LifecycleEventText, LoadRowText, + ModelResultText, NodeResultText, PythonNodeText, SkipText, TestResultText, TestStepText, +}; +use sqlbuild_core::progress::main::format_expectation_detail::format_expectation_detail; + +use crate::bindings::_helpers::cli::progress_text::expectation_step; +use crate::bindings::_helpers::plan_output::display::{cursor_bound_display, inclusive_cursor_end}; +use crate::bindings::_helpers::sql_test_command::test_command::test_label; + +const SQL_EVENT: &str = "sql"; +const PRE_HOOKS: &str = "pre_hooks"; +const MEASUREMENT: &str = "measurement"; +const FAILED: &str = "failed"; + +pub(crate) fn text(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + Ok(value.getattr(name)?.str()?.to_string()) +} + +pub(crate) fn optional(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + let field: Bound<'_, PyAny> = value.getattr(name)?; + if field.is_none() { + return Ok(None); + } + Ok(Some(field.str()?.to_string())) +} + +fn texts(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + value + .getattr(name)? + .try_iter()? + .map(|item| Ok(item?.str()?.to_string())) + .collect() +} + +fn items<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult>> { + value.getattr(name)?.try_iter()?.collect() +} + +fn duration(value: &Bound<'_, PyAny>) -> PyResult> { + value.getattr("duration_ms")?.extract() +} + +fn error(value: &Bound<'_, PyAny>) -> PyResult { + Ok(ErrorText { + code: optional(value, "error_code")?, + message: optional(value, "error_message")?, + help: optional(value, "error_help")?, + }) +} + +fn skip(value: &Bound<'_, PyAny>) -> PyResult { + Ok(SkipText { + mode: optional(value, "skip_mode")?, + reason: optional(value, "skip_reason")?, + }) +} + +fn events(value: &Bound<'_, PyAny>) -> PyResult> { + let mut events: Vec = Vec::new(); + for event in items(value, "lifecycle_events")? { + let content: String = event.getattr("content")?.extract()?; + events.push(if text(&event, "kind")? == SQL_EVENT { + LifecycleEventText::Sql(content) + } else { + LifecycleEventText::Log(content) + }); + } + Ok(events) +} + +pub(crate) fn audit_text(audit: &Bound<'_, PyAny>) -> PyResult { + Ok(AuditResultText { + audit_name: text(audit, "audit_name")?, + attached_column_name: optional(audit, "attached_column_name")?, + attached_target_name: optional(audit, "attached_target_name")?, + run_scope_phase: text(audit, "run_scope_phase")?, + outcome: text(audit, "outcome")?, + row_count: audit.getattr("row_count")?.extract()?, + reused: audit.getattr("reused")?.is_truthy()?, + executed_sql: optional(audit, "executed_sql")?, + measurement: text(audit, "evaluation_mode")? == MEASUREMENT, + measured_value: text(audit, "measured_value")?, + execution_error: optional(audit, "execution_error")?, + }) +} + +fn audits(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + items(value, name)?.iter().map(audit_text).collect() +} + +fn hook_text(hook: &Bound<'_, PyAny>) -> PyResult { + Ok(HookResultText { + pre_hook: text(hook, "phase")? == PRE_HOOKS, + hook_type: text(hook, "hook_type")?, + label: text(hook, "label")?, + status: text(hook, "status")?, + skip: skip(hook)?, + }) +} + +fn cursor_range(model: &Bound<'_, PyAny>) -> PyResult> { + let start: Bound<'_, PyAny> = model.getattr("cursor_range_start")?; + let end: Bound<'_, PyAny> = model.getattr("cursor_range_end")?; + if start.is_none() || end.is_none() || model.getattr("empty_cursor_inputs")?.is_truthy()? { + return Ok(None); + } + let cursor_type: Bound<'_, PyAny> = model.getattr("cursor_type")?; + let cursor_grain: Bound<'_, PyAny> = model.getattr("cursor_grain")?; + Ok(Some(( + cursor_bound_display(&start, &cursor_type, &cursor_grain)?, + inclusive_cursor_end(&end, &cursor_type, &cursor_grain)?, + ))) +} + +pub(crate) fn model_text(model: &Bound<'_, PyAny>) -> PyResult { + Ok(ModelResultText { + name: text(model, "model_name")?, + status: text(model, "status")?, + failed_phase: optional(model, "failed_phase")?, + staging_relation: optional(model, "staging_relation")?, + duration_ms: duration(model)?, + batch_count: model.getattr("batch_count")?.extract()?, + batch_size: optional(model, "batch_size")?, + rows_affected: model.getattr("rows_affected")?.extract()?, + cursor_range: cursor_range(model)?, + empty_cursor_inputs: texts(model, "empty_cursor_inputs")?, + waiting_on_empty_inputs: model.getattr("waiting_on_empty_inputs")?.is_truthy()?, + recovery_batch_count: model + .getattr("microbatch_recovery_batch_count")? + .extract()?, + synthetic_completion_count: model + .getattr("microbatch_synthetic_completion_count")? + .extract()?, + replay_requirement_state: optional(model, "microbatch_replay_requirement_state")?, + audits: audits(model, "audit_results")?, + warnings: texts(model, "warning_messages")?, + events: events(model)?, + hooks: items(model, "hook_results")? + .iter() + .map(hook_text) + .collect::>()?, + skip: skip(model)?, + error: error(model)?, + }) +} + +pub(crate) fn seed_text(seed: &Bound<'_, PyAny>) -> PyResult { + Ok(NodeResultText { + name: text(seed, "seed_name")?, + kind: "seed".to_owned(), + status: text(seed, "status")?, + duration_ms: duration(seed)?, + error: error(seed)?, + warnings: Vec::new(), + }) +} + +pub(crate) fn function_text(function: &Bound<'_, PyAny>) -> PyResult { + Ok(NodeResultText { + name: text(function, "function_name")?, + kind: text(function, "function_kind")?, + status: text(function, "status")?, + duration_ms: duration(function)?, + error: error(function)?, + warnings: texts(function, "warning_messages")?, + }) +} + +pub(crate) fn load_text(load: &Bound<'_, PyAny>) -> PyResult { + Ok(LoadRowText { + source_name: text(load, "source_name")?, + loader_name: text(load, "loader_name")?, + resource_kind: text(load, "resource_kind")?, + status: text(load, "status")?, + rows_loaded: load.getattr("rows_loaded")?.extract()?, + duration_ms: duration(load)?, + events: events(load)?, + skip: skip(load)?, + error_message: optional(load, "error_message")?, + warnings: texts(load, "warning_messages")?, + }) +} + +pub(crate) fn test_text(test: &Bound<'_, PyAny>) -> PyResult { + let mut steps: Vec = Vec::new(); + for step in items(test, "step_results")? { + steps.push(TestStepText { + model_name: text(&step, "model_name")?, + outcome: text(&step, "outcome")?, + detail: format_expectation_detail(&expectation_step(&step)?), + }); + } + Ok(TestResultText { + label: test_label(test, "test_name")?, + outcome: text(test, "outcome")?, + steps, + error: error(test)?, + }) +} + +pub(crate) fn python_node_text(node: &Bound<'_, PyAny>) -> PyResult { + Ok(PythonNodeText { + name: text(node, "node_name")?, + kind: text(node, "kind")?, + status: text(node, "status")?, + error_message: optional(node, "error_message")?, + warnings: texts(node, "warning_messages")?, + }) +} + +pub(crate) fn build_text(result: &Bound<'_, PyAny>) -> PyResult { + Ok(BuildResultText { + failed: text(result, "status")? == FAILED, + warning_count: result.getattr("warning_count")?.extract()?, + insufficient_count: result.getattr("insufficient_count")?.extract()?, + models: items(result, "model_results")? + .iter() + .map(model_text) + .collect::>()?, + seeds: items(result, "seed_results")? + .iter() + .map(seed_text) + .collect::>()?, + functions: items(result, "function_results")? + .iter() + .map(function_text) + .collect::>()?, + loads: items(result, "load_results")? + .iter() + .map(load_text) + .collect::>()?, + tests: items(result, "test_results")? + .iter() + .map(test_text) + .collect::>()?, + source_audits: audits(result, "source_audit_results")?, + end_audits: audits(result, "end_audit_results")?, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs new file mode 100644 index 0000000000..65a1bb16ac --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs @@ -0,0 +1,118 @@ +//! Terminal width, the shared transient line coordinator and spinner threads of progress reporters. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const COLUMNS_MODULE: &str = "sqlbuild.presentation.main.terminal_columns"; +const LINES_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; +const SPINNER_JOIN_SECONDS: f64 = 0.2; +pub(crate) const SPINNER_TICK_SECONDS: f64 = 0.1; +pub(crate) const CLEAR_LINE: &str = "\r\u{1b}[K"; +pub(crate) const HIDE_CURSOR: &str = "\u{1b}[?25l"; +pub(crate) const SHOW_CURSOR: &str = "\u{1b}[?25h"; +pub(crate) const SPINNER_FRAMES: [&str; 10] = [ + "\u{280b}", "\u{2819}", "\u{2839}", "\u{2838}", "\u{283c}", "\u{2834}", "\u{2826}", "\u{2827}", + "\u{2807}", "\u{280f}", +]; + +/// The current terminal width. +pub(crate) fn terminal_columns(py: Python<'_>) -> PyResult { + let columns: i64 = imported(py, COLUMNS_MODULE, "terminal_columns")? + .call0()? + .extract()?; + Ok(usize::try_from(columns).unwrap_or(0)) +} + +/// The process-wide coordinator of the one live transient line. +pub(crate) fn shared_lines(py: Python<'_>) -> PyResult> { + imported(py, LINES_MODULE, "shared_transient_line_coordinator")?.call0() +} + +/// Whether `stream` reports a terminal. +pub(crate) fn tty_flag(stream: &Bound<'_, PyAny>) -> PyResult { + if !stream.hasattr("isatty")? { + return Ok(false); + } + stream.call_method0("isatty")?.is_truthy() +} + +/// The current native progress projector, if a projection is active. +pub(crate) fn projector(py: Python<'_>) -> PyResult>> { + let projector: Bound<'_, PyAny> = + imported(py, "sqlbuild._native", "current_native_progress_projector")?.call0()?; + Ok((!projector.is_none()).then(|| projector.unbind())) +} + +/// The projector's canonical duration of a finished resource, `None` while it is pending. +pub(crate) fn consume_terminal( + projector: &Bound<'_, PyAny>, + resource_name: &str, + resource_id: Option<&str>, +) -> PyResult> { + let options: Bound<'_, PyDict> = PyDict::new(projector.py()); + options.set_item("resource_name", resource_name)?; + options.set_item("resource_id", resource_id)?; + projector + .call_method("consume_resource_terminal", (), Some(&options))? + .extract() +} + +/// Claim or release the live line for `owner`. +pub(crate) fn claim( + lines: &Bound<'_, PyAny>, + stream: &Bound<'_, PyAny>, + owner: &Bound<'_, PyAny>, +) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(lines.py()); + options.set_item("stream", stream)?; + options.set_item("owner", owner)?; + lines.call_method("claim", (), Some(&options))?; + Ok(()) +} + +pub(crate) fn release(lines: &Bound<'_, PyAny>, owner: &Bound<'_, PyAny>) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(lines.py()); + options.set_item("owner", owner)?; + lines.call_method("release", (), Some(&options))?; + Ok(()) +} + +pub(crate) fn write(stream: &Bound<'_, PyAny>, text: &str) -> PyResult<()> { + stream.call_method1("write", (text,))?; + Ok(()) +} + +pub(crate) fn flush(stream: &Bound<'_, PyAny>) -> PyResult<()> { + stream.call_method0("flush")?; + Ok(()) +} + +/// Start a daemon thread running `owner._spin_until_stopped(event)`; return the event and thread. +pub(crate) fn start_spinner<'py>( + owner: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = owner.py(); + let event: Bound<'py, PyAny> = imported(py, "threading", "Event")?.call0()?; + let thread: Bound<'py, PyAny> = call_with( + &imported(py, "threading", "Thread")?, + &[ + ("target", owner.getattr("_spin_until_stopped")?), + ("args", PyTuple::new(py, [&event])?.into_any()), + ("daemon", PyBool::new(py, true).to_owned().into_any()), + ], + )?; + thread.call_method0("start")?; + Ok((event, thread)) +} + +/// Join a stopped spinner thread briefly if it is still alive. +pub(crate) fn join_spinner(thread: &Bound<'_, PyAny>) -> PyResult<()> { + if thread.call_method0("is_alive")?.is_truthy()? { + let options: Bound<'_, PyDict> = PyDict::new(thread.py()); + options.set_item("timeout", SPINNER_JOIN_SECONDS)?; + thread.call_method("join", (), Some(&options))?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs new file mode 100644 index 0000000000..bbf3d306bf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs @@ -0,0 +1,308 @@ +//! One transient status line with a spinner on a terminal and plain lines elsewhere. + +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::progress::main::format_status_spinner_line::format_status_spinner_line; + +use crate::bindings::_helpers::build_progress::terminal::{ + CLEAR_LINE, HIDE_CURSOR, SHOW_CURSOR, SPINNER_FRAMES, SPINNER_TICK_SECONDS, claim, flush, + join_spinner, release, shared_lines, start_spinner, terminal_columns, tty_flag, write, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{imported, within}; + +const SPINNERS_DISABLED_MODULE: &str = "sqlbuild.presentation.main._progress_spinners_disabled"; +const PREFLIGHT_COMPLETION_PREFIX: &str = "Prepared "; + +/// Mutable reporter state; only held briefly and never across Python calls. +#[derive(Default)] +struct StatusState { + message: Option, + active: bool, + drawn: bool, + frame: usize, + stop_event: Option>, + threads: Vec>, + exit_hook_registered: bool, +} + +/// Render one transient status line with optional spinner updates. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct TransientStatusReporter { + stream: Py, + lines: Py, + lock: Py, + style: CliStyle, + enabled: bool, + state: Mutex, +} + +impl TransientStatusReporter { + fn state(&self) -> MutexGuard<'_, StatusState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn locked(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { + within(self.lock.bind(py), |_| body()) + } + + fn write_persistent(&self, py: Python<'_>, text: &str) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("stream", self.stream.bind(py))?; + options.set_item("text", text)?; + self.lines + .bind(py) + .call_method("write_persistent", (), Some(&options))?; + Ok(()) + } + + fn write_message(&self, py: Python<'_>, message: &str, dim_output: bool) -> PyResult<()> { + let formatted: String = if dim_output { + self.style.muted(message) + } else { + message.to_owned() + }; + self.write_persistent(py, &format!("{formatted}\n")) + } + + fn draw(&self, py: Python<'_>) -> PyResult<()> { + let (message, frame) = { + let mut state = self.state(); + let Some(message) = state.message.clone().filter(|_| state.drawn) else { + return Ok(()); + }; + let frame: usize = state.frame; + state.frame = (frame + 1) % SPINNER_FRAMES.len(); + (message, frame) + }; + let line: String = format_status_spinner_line( + SPINNER_FRAMES[frame], + &message, + terminal_columns(py)?, + self.style, + ); + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, &format!("{CLEAR_LINE}{line}"))?; + flush(stream) + } + + fn enter_status(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + let register_hook: bool = + !std::mem::replace(&mut reporter.state().exit_hook_registered, true); + if register_hook { + imported(py, "atexit", "register")?.call1((slf.getattr("close")?,))?; + } + let stream: &Bound<'_, PyAny> = reporter.stream.bind(py); + write(stream, HIDE_CURSOR)?; + reporter.state().drawn = true; + reporter.draw(py)?; + let (event, thread) = start_spinner(slf.as_any())?; + let mut state = reporter.state(); + state.stop_event = Some(event.unbind()); + state.threads.push(thread.unbind()); + Ok(()) + } + + fn close_status(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + reporter.locked(py, || { + release(reporter.lines.bind(py), slf.as_any())?; + reporter.exit_status(py)?; + let mut state = reporter.state(); + state.message = None; + state.active = false; + Ok(()) + }) + } + + fn exit_status(&self, py: Python<'_>) -> PyResult<()> { + let (drawn, event) = { + let mut state = self.state(); + (std::mem::take(&mut state.drawn), state.stop_event.take()) + }; + if let Some(event) = event { + event.bind(py).call_method0("set")?; + } + if drawn { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, &format!("{CLEAR_LINE}{SHOW_CURSOR}"))?; + flush(stream)?; + } + Ok(()) + } +} + +#[pymethods] +impl TransientStatusReporter { + #[new] + #[pyo3(signature = (*, stream, use_color = false, enabled = true))] + fn new( + py: Python<'_>, + stream: &Bound<'_, PyAny>, + use_color: bool, + enabled: bool, + ) -> PyResult { + let lines: Bound<'_, PyAny> = shared_lines(py)?; + let spinners_disabled: bool = + imported(py, SPINNERS_DISABLED_MODULE, "progress_spinners_disabled")? + .call0()? + .is_truthy()?; + Ok(Self { + enabled: enabled && !spinners_disabled && tty_flag(stream)?, + stream: stream.clone().unbind(), + lock: lines.getattr("lock")?.unbind(), + lines: lines.unbind(), + style: CliStyle { use_color }, + state: Mutex::new(StatusState::default()), + }) + } + + fn start(slf: &Bound<'_, Self>, message: String) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + if reporter.enabled { + return reporter.locked(py, || { + Self::close_status(slf)?; + reporter.state().message = Some(message); + Self::enter_status(slf)?; + claim( + reporter.lines.bind(py), + reporter.stream.bind(py), + slf.as_any(), + )?; + reporter.state().active = true; + Ok(()) + }); + } + reporter.state().active = true; + reporter.write_message(py, &message, true) + } + + fn update(&self, py: Python<'_>, message: String) -> PyResult<()> { + if self.enabled { + let updated: bool = self.locked(py, || { + let drawn: bool = { + let mut state = self.state(); + if state.message.is_none() { + return Ok(false); + } + state.message = Some(message.clone()); + state.drawn + }; + if drawn { + self.draw(py)?; + } + Ok(true) + })?; + if updated { + return Ok(()); + } + } + self.write_message(py, &message, true) + } + + #[pyo3(signature = (*, message, blank_line_after = false))] + fn complete(slf: &Bound<'_, Self>, message: &str, blank_line_after: bool) -> PyResult<()> { + let py: Python<'_> = slf.py(); + Self::close(slf)?; + slf.get().write_message(py, message, true)?; + if blank_line_after { + slf.get().write_blank_line(py)?; + } + Ok(()) + } + + /// Complete with a pre-styled message written verbatim. + #[pyo3(signature = (*, message, blank_line_after = false))] + fn complete_styled( + slf: &Bound<'_, Self>, + message: &str, + blank_line_after: bool, + ) -> PyResult<()> { + Self::close(slf)?; + let blank: &str = if blank_line_after { "\n" } else { "" }; + slf.get() + .write_persistent(slf.py(), &format!("{message}\n{blank}")) + } + + fn error(slf: &Bound<'_, Self>, message: &str) -> PyResult<()> { + Self::close(slf)?; + slf.get().write_message(slf.py(), message, false) + } + + fn close(slf: &Bound<'_, Self>) -> PyResult<()> { + Self::close_status(slf)?; + let threads: Vec> = std::mem::take(&mut slf.get().state().threads); + for thread in threads { + join_spinner(thread.bind(slf.py()))?; + } + Ok(()) + } + + /// Erase the live status so a persistent line can be written in its place. + fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.exit_status(py) + } + + /// Restart the live status below persistent output written while it was active. + fn redraw_transient_line(slf: &Bound<'_, Self>) -> PyResult<()> { + let redraw: bool = { + let state = slf.get().state(); + state.message.is_some() && !state.drawn + }; + if redraw { + Self::enter_status(slf)?; + } + Ok(()) + } + + /// Render one preflight progress update. + fn report_preflight_progress(slf: &Bound<'_, Self>, message: String) -> PyResult<()> { + if message.starts_with(PREFLIGHT_COMPLETION_PREFIX) { + return Self::complete(slf, &message, true); + } + if !slf.get().state().active { + return Self::start(slf, message); + } + slf.get().update(slf.py(), message) + } + + fn write_blank_line(&self, py: Python<'_>) -> PyResult<()> { + self.write_persistent(py, "\n") + } + + fn _spin_until_stopped(&self, py: Python<'_>, stop_event: &Bound<'_, PyAny>) -> PyResult<()> { + while !stop_event + .call_method1("wait", (SPINNER_TICK_SECONDS,))? + .is_truthy()? + { + self.locked(py, || { + if stop_event.call_method0("is_set")?.is_truthy()? { + return Ok(()); + } + self.draw(py) + })?; + } + Ok(()) + } +} + +/// A native status reporter writing to `stream`. +pub(crate) fn status_reporter<'py>( + stream: &Bound<'py, PyAny>, + use_color: bool, +) -> PyResult> { + let reporter: TransientStatusReporter = + TransientStatusReporter::new(stream.py(), stream, use_color, true)?; + Ok(Bound::new(stream.py(), reporter)?.into_any()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 6d6a88e47f..609fbd9225 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -25,7 +25,7 @@ const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_ const CHECK_CORE_MODULE: &str = "sqlbuild.cli.commands._helpers.check.core"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main._static_command"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const RELATION_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._relation_plan"; const SELECTOR_PARSE_MODULE: &str = "sqlbuild.compiler.planner.main.selection.selector_parse"; @@ -33,7 +33,6 @@ const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const PYTHON_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; -const PYTHON_LIFECYCLE_MODULE: &str = "sqlbuild.compiler.python_nodes.main._run_lifecycle"; const PYTHON_NODE_MODELS: &str = "sqlbuild.compiler.python_nodes.models"; const PYTHON_NODE_TYPES: &str = "sqlbuild.compiler.python_nodes.types"; const EXECUTOR_MODELS: &str = "sqlbuild.executor.python_nodes.models"; @@ -337,11 +336,7 @@ fn prepare<'py>( ], )?; let lifecycle_plan: Bound<'py, PyAny> = call_with( - &imported( - py, - PYTHON_LIFECYCLE_MODULE, - "build_python_sql_run_lifecycle", - )?, + &imported(py, NATIVE_MODULE, "build_python_sql_run_lifecycle")?, &[ ("selection", selection), ("python_graph", python_graph.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs index e83139b0f1..f73973bdc3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs @@ -10,13 +10,12 @@ use sqlbuild_core::progress::main::connection_phase_line::connection_phase_line; use sqlbuild_core::progress::main::connection_start_message::connection_start_message; use sqlbuild_core::progress::main::is_planning_completion_message::is_planning_completion_message; +use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::cli::progress::{ NativeProgressProjector, current_projector, projector_operation_active, write_persistent, }; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; -const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; -const STATUS_REPORTER_CLASS: &str = "TransientStatusReporter"; const CONNECTION_OPTIONS: (&[&str], &[&str]) = ( &["adapter_name", "stream"], &[ @@ -133,13 +132,7 @@ impl PlanningProgressReporter { #[new] #[pyo3(signature = (*, stream, use_color = false))] fn new(py: Python<'_>, stream: &Bound<'_, PyAny>, use_color: bool) -> PyResult { - let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("stream", stream)?; - keywords.set_item("use_color", use_color)?; - let status: Bound<'_, PyAny> = py - .import(STATUS_REPORTER_MODULE)? - .getattr(STATUS_REPORTER_CLASS)? - .call((), Some(&keywords))?; + let status: Bound<'_, PyAny> = status_reporter(stream, use_color)?; Ok(Self { status: status.unbind(), active: AtomicBool::new(false), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs index f5e80e4a5f..1df6d8f56d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs @@ -63,7 +63,7 @@ fn difference_samples(samples: &Bound<'_, PyAny>) -> PyResult) -> PyResult { +pub(crate) fn expectation_step(step: &Bound<'_, PyAny>) -> PyResult { let mut column_differences: Vec<(String, String, String)> = Vec::new(); for difference in step.getattr("column_differences")?.try_iter()? { let difference: Bound<'_, PyAny> = difference?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 58be3c4671..7e12aa65a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -4,10 +4,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::cli::{argv, entry, progress, progress_reporters, progress_text}; use crate::bindings::_helpers::{ - audit_command, build_command, check, clone, command_runtime, contract_command, cost_command, - cost_report, dbt_command, dbt_events, dbt_interop, debug_command, diff_report, format_command, - freshness, init_command, janitor, load, presentation_text, query_command, result_output, - scenario_command, seed_command, sql_test_command, workspace_commands, + audit_command, build_command, build_progress, check, clone, command_runtime, contract_command, + cost_command, cost_report, dbt_command, dbt_events, dbt_interop, debug_command, diff_report, + format_command, freshness, init_command, janitor, load, presentation_text, query_command, + result_output, scenario_command, seed_command, sql_test_command, workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -34,6 +34,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { seed_command::registration::register(module)?; audit_command::registration::register(module)?; build_command::registration::register(module)?; + build_progress::registration::register(module)?; query_command::registration::register(module)?; scenario_command::registration::register(module)?; contract_command::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index 50470d396a..d131b3336b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -21,6 +21,7 @@ use sqlbuild_core::clone_text::models::CloneItemText; use crate::bindings::_helpers::clone::clone_execution::{ destination_entry_count, execute_clone_plan, resource_types_by_name, }; +use crate::bindings::_helpers::clone::clone_pipeline::{ClonePipeline, prepared_clone_pipeline}; use crate::bindings::_helpers::clone::clone_progress::{clone_item_text, write_line}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; @@ -28,12 +29,11 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; -const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const TARGET_NAME_MODULE: &str = "sqlbuild._native"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; -const CLONE_PIPELINE_MODULE: &str = "sqlbuild.compiler.pipeline.main._clone"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const NO_DESTINATION: (&str, &str) = ( @@ -265,21 +265,28 @@ fn prepare<'py>( &imported(py, PIPELINE_MODELS, "ClonePipelineConnection")?, &[("config", config.clone()), ("handle", connection.clone())], )?; - let pipeline: Bound<'py, PyAny> = call_with( - &imported(py, CLONE_PIPELINE_MODULE, "run_clone_pipeline")?, + let options: Bound<'py, PyAny> = call_with( + &imported(py, PIPELINE_MODELS, "ClonePipelineOptions")?, &[ - ("discovered_inputs", invocation.discovered.clone()), - ("adapter", invocation.adapter.clone()), - ("origin_target_name", request.getattr("origin_target_name")?), - ("destination_target_name", text(py, &invocation.destination)), ("no_sql_validation", request.getattr("no_sql_validation")?), ("select", request.getattr("select")?), ("exclude", request.getattr("exclude")?), ("cli_vars", request.getattr("cli_vars")?), - ("destination_connection", destination_connection), - ("external_sql_reference_resolver", resolver), ], )?; + let pipeline: Bound<'py, PyAny> = prepared_clone_pipeline( + &ClonePipeline { + discovered: invocation.discovered.clone(), + adapter: invocation.adapter.clone(), + connection: destination_connection, + options, + resolver, + }, + ( + &request.getattr("origin_target_name")?, + &text(py, &invocation.destination), + ), + )?; progress.call_method1( "on_progress", (format!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs index 742ff72c25..c5f6e01005 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs @@ -23,7 +23,7 @@ const TRANSIENT_MODELS_MODULE: &str = "sqlbuild.executor.clone.main.build_destination_transient_models"; const FINGERPRINTING_MODULE: &str = "sqlbuild.executor.clone.main.fingerprinting"; const EXECUTE_MODULE: &str = "sqlbuild.executor.clone.main.execute"; -const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const DESTINATION_ENTRY_GROUPS: [&str; 4] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs new file mode 100644 index 0000000000..1677450c25 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs @@ -0,0 +1,167 @@ +//! Compile the origin and destination environments of a clone and plan it. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; +const CLONE_PLANNING_MODULE: &str = "sqlbuild.compiler.planner.main.clone._clone"; +/// The `ClonePipelineResult` fields after the two projects, in `run_clone_planning` order. +const PLANNED_FIELDS: [&str; 8] = [ + "clone_plan", + "destination_source_entries", + "destination_model_entries", + "destination_seed_entries", + "destination_function_entries", + "origin_model_entries", + "origin_seed_entries", + "origin_source_entries", +]; + +/// The discovered project, adapter, destination connection, options and reference resolver. +pub(crate) struct ClonePipeline<'py> { + pub(crate) discovered: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) connection: Bound<'py, PyAny>, + pub(crate) options: Bound<'py, PyAny>, + pub(crate) resolver: Bound<'py, PyAny>, +} + +/// The compiled project of one target environment. +fn compiled_environment<'py>( + pipeline: &ClonePipeline<'py>, + target_name: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = target_name.py(); + let options: &Bound<'py, PyAny> = &pipeline.options; + call_with( + &imported(py, NATIVE_MODULE, "build_compiled_project")?, + &[ + ("discovered_inputs", pipeline.discovered.clone()), + ("adapter", pipeline.adapter.clone()), + ("selected_target", target_name.clone()), + ( + "resolved_connection", + pipeline.connection.getattr("config")?, + ), + ("no_sql_validation", options.getattr("no_sql_validation")?), + ( + "analysis_selection", + call_with( + &imported(py, COMPILE_MODELS_MODULE, "CompileAnalysisSelection")?, + &[("no_cache", options.getattr("no_cache")?)], + )?, + ), + ("cli_vars", options.getattr("cli_vars")?), + ("external_sql_reference_resolver", pipeline.resolver.clone()), + ], + ) +} + +/// The target whose sources a target reads: its `defer_sources_to`, or itself. +fn read_target_name<'py>( + discovered: &Bound<'py, PyAny>, + target_name: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = target_name.py(); + let deferred: Bound<'py, PyAny> = resolve_target_config( + py, + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, + target_name, + )? + .getattr("defer_sources_to")?; + if deferred.is_truthy()? { + return Ok(deferred); + } + Ok(target_name.clone()) +} + +/// The project its sources are read from: `project` itself unless sources defer elsewhere. +fn source_project<'py>( + pipeline: &ClonePipeline<'py>, + project: &Bound<'py, PyAny>, + target_name: &Bound<'py, PyAny>, +) -> PyResult> { + let read_target: Bound<'py, PyAny> = read_target_name(&pipeline.discovered, target_name)?; + if read_target.eq(target_name)? { + return Ok(project.clone()); + } + compiled_environment(pipeline, &read_target) +} + +/// Compile both environments, plan the clone and return the `ClonePipelineResult`. +pub(crate) fn prepared_clone_pipeline<'py>( + pipeline: &ClonePipeline<'py>, + (origin_target, destination_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = origin_target.py(); + let origin_project: Bound<'py, PyAny> = compiled_environment(pipeline, origin_target)?; + let destination_project: Bound<'py, PyAny> = + compiled_environment(pipeline, destination_target)?; + let origin_source_project: Bound<'py, PyAny> = + source_project(pipeline, &origin_project, origin_target)?; + let destination_source_project: Bound<'py, PyAny> = + source_project(pipeline, &destination_project, destination_target)?; + let planned: Bound<'py, PyTuple> = call_with( + &imported(py, CLONE_PLANNING_MODULE, "run_clone_planning")?, + &[ + ("project", destination_project.clone()), + ("select", pipeline.options.getattr("select")?), + ("exclude", pipeline.options.getattr("exclude")?), + ("adapter", pipeline.adapter.clone()), + ("connection", pipeline.connection.getattr("handle")?), + ("origin_project", origin_project.clone()), + ("origin_source_project", origin_source_project), + ("destination_source_project", destination_source_project), + ], + )? + .downcast_into()?; + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("origin_project", origin_project), + ("destination_project", destination_project), + ]; + for (index, name) in PLANNED_FIELDS.iter().enumerate() { + fields.push((*name, planned.get_item(index)?)); + } + call_with( + &imported(py, PIPELINE_MODELS_MODULE, "ClonePipelineResult")?, + &fields, + ) +} + +/// Run a clone pipeline with explicit compilation controls. +#[pyfunction] +#[pyo3(signature = (**request))] +fn run_clone_pipeline_with_options<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request: Request<'_, 'py> = Request::new("run_clone_pipeline_with_options", request)?; + let py: Python<'py> = request.py(); + prepared_clone_pipeline( + &ClonePipeline { + discovered: request.get("discovered_inputs")?, + adapter: request.get("adapter")?, + connection: request.get("destination_connection")?, + options: request.get("options")?, + resolver: request + .optional("external_sql_reference_resolver")? + .unwrap_or_else(|| py.None().into_bound(py)), + }, + ( + &request.get("origin_target_name")?, + &request.get("destination_target_name")?, + ), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_clone_pipeline_with_options, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/mod.rs index 6ac5b67d26..9a3293f7cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod clone_command; pub(crate) mod clone_execution; +pub(crate) mod clone_pipeline; pub(crate) mod clone_progress; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs index 9df257097e..bca0112872 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs @@ -2,8 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::clone::clone_command; +use crate::bindings::_helpers::clone::{clone_command, clone_pipeline}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - clone_command::register(module) + clone_command::register(module)?; + clone_pipeline::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 7ee3a0cc7d..0ab1a951c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -20,10 +20,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const DBT_PROFILE_MODULE: &str = "sqlbuild.integrations.dbt.main.profile.profile_connection"; const DBT_ERRORS_MODULE: &str = "sqlbuild.integrations.dbt.exceptions"; -const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; -const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; -const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild._native"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; +const TARGET_NAME_MODULE: &str = "sqlbuild._native"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const CONNECTION_OPTIONS: (&[&str], &[&str]) = ( &["raw_config", "project_dir", "adapter_name"], &["discovered_inputs", "cli_vars", "selected_target"], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index daff68d0bd..e02035283c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyMapping, PyString}; +use pyo3::types::{PyDict, PyDictMethods, PyMapping, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_config::command_connection::constants::{ CLI_WAREHOUSE_SOURCE, CONNECTION_DEFAULT_WAREHOUSE_SOURCE, CONNECTION_WAREHOUSE_KEY, @@ -16,6 +16,8 @@ use sqlbuild_config::command_connection::main::unsupported_cli_warehouse::unsupp use sqlbuild_config::command_connection::models::AuthoredWarehouse; use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_model_config::templates::models::TemplateOptions; + use crate::bindings::_helpers::command_runtime::runtime_errors::{ expected_resolution_error, raised, }; @@ -23,14 +25,14 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::discovery::configuration::discover_project_configuration; +use crate::bindings::_helpers::model_config::template_data::expanded_template_data; const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; const SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; -const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const TARGET_NAME_MODULE: &str = "sqlbuild._native"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main._session_warehouse_support"; -const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; -const EXPAND_TEMPLATE_MODULE: &str = "sqlbuild.compiler.compile.main.expand_template_data"; +const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild._native"; const RESOLVE_OPTIONS: (&[&str], &[&str]) = ( &[ "discovered_inputs", @@ -182,32 +184,17 @@ fn expand_authored( ("cli_vars", cli_vars.clone()), ], )?; - let expanded: Bound<'_, PyAny> = call_with( - &imported(py, EXPAND_TEMPLATE_MODULE, "expand_template_data")?, - &[ - ( - "value", - PyString::new(py, &found.authored.warehouse).into_any(), - ), - ("variables", runtime.get_item(1)?), - ("context_values", PyDict::new(py).into_any()), - ( - "context_label", - PyString::new(py, &found.authored.context_label()).into_any(), - ), - ( - "allow_context", - PyBool::new(py, false).to_owned().into_any(), - ), - ( - "preserve_context_tokens", - PyBool::new(py, false).to_owned().into_any(), - ), - ( - "preserve_unknown_context", - PyBool::new(py, false).to_owned().into_any(), - ), - ], + let expanded: Bound<'_, PyAny> = expanded_template_data( + PyString::new(py, &found.authored.warehouse).as_any(), + (runtime.get_item(1)?.downcast()?, &PyDict::new(py)), + ( + TemplateOptions { + allow_context: false, + preserve_context_tokens: false, + preserve_unknown_context: false, + }, + &found.authored.context_label(), + ), )?; let warehouse: String = expanded.str()?.extract()?; check_expanded_warehouse(&found.authored, &warehouse).map_err(|error| raised(py, error))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs index cb2e2c978e..f3769dc70d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -5,16 +5,17 @@ use pyo3::pyfunction; use pyo3::types::PyTuple; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::render::declaration_values::declaration_scope_resolver; +use crate::bindings::_helpers::scopes::scope_index::open_native_scope_index; +use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; +use crate::bindings::_helpers::scopes::scope_validation::validate_scope_index; const NATIVE_MODULE: &str = "sqlbuild._native"; const SCAN_CACHE_MODULE: &str = "sqlbuild.compiler.compile.classes.sql_test_scan_cache"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; -const OPEN_INDEX_MODULE: &str = "sqlbuild.compiler.scopes.main._open_native_scope_index"; -const VALIDATE_INDEX_MODULE: &str = "sqlbuild.compiler.scopes.main._validate_scope_index"; fn compile_error(py: Python<'_>, message: Bound<'_, PyAny>) -> PyResult { Ok(PyErr::from_value( @@ -29,15 +30,8 @@ fn scope_build<'py>( (index, lookup, scan_cache): (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>), ) -> PyResult> { let py: Python<'py> = discovered.py(); - let resolver: Bound<'py, PyAny> = call_with( - &imported(py, DECLARATIONS_MODULE, "build_declaration_scope_resolver")?, - &[ - ("discovered_inputs", discovered.clone()), - ("scope_index", index.clone()), - ("loaded_macros", loaded_macros.clone()), - ("lookup", lookup), - ], - )?; + let resolver: Bound<'py, PyAny> = + declaration_scope_resolver(discovered, &index, (loaded_macros, &lookup))?; call_with( &imported(py, COMPILE_MODELS_MODULE, "DeclarationScopeBuild")?, &[ @@ -73,17 +67,12 @@ pub(crate) fn build_declaration_scope<'py>( &scenario_files, sql_lexical_syntax.getattr("native_mapping")?, ))?; - let native: Bound<'py, PyAny> = call_with( - &imported(py, OPEN_INDEX_MODULE, "open_native_scope_index")?, - &[ - ("discovered_inputs", discovered_inputs.clone()), - ("loaded_macros", loaded_macros.clone()), - ], - )?; - let validated: PyResult> = call_with( - &imported(py, VALIDATE_INDEX_MODULE, "validate_scope_index")?, - &[("index", native.getattr("index")?)], - ); + let native: Bound<'py, PyAny> = Bound::new( + py, + open_native_scope_index(py, discovered_inputs, loaded_macros, None)?, + )? + .into_any(); + let validated: Result<(), PyErr> = validate_scope_index(&native.getattr("index")?); if let Err(error) = validated { if !error.is_instance( py, @@ -127,8 +116,7 @@ pub(crate) fn build_declaration_scope<'py>( } } let index: Bound<'py, PyAny> = native.call_method0("index_with_relationships")?; - let lookup: Bound<'py, PyAny> = - imported(py, NATIVE_MODULE, "build_scope_lookup")?.call1((&index,))?; + let lookup: Bound<'py, PyAny> = build_scope_lookup(py, &index)?.into_bound(py); scope_build( discovered_inputs, loaded_macros, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs index bff1441a51..96bf6589cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs @@ -2,13 +2,20 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyfunction; -use pyo3::types::{PyBool, PyDict, PyList, PyString}; +use pyo3::types::{PyDict, PyList, PyTuple}; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use sqlbuild_model_config::templates::models::TemplateOptions; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::contracts::spec_resolution::{ + resolve_target_config, resolve_target_name, +}; +use crate::bindings::_helpers::model_config::template_data::{ + expanded_effective_vars, expanded_template_data, run_id, +}; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const TEMPLATING_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.templating"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; const CONNECTION_LABEL: &str = "effective connection"; @@ -31,21 +38,17 @@ pub(crate) fn build_effective_connection<'py>( .call1((error.unwrap_or_default(),))?; return Err(PyErr::from_value(raised)); } - let falsy = || PyBool::new(py, false).to_owned().into_any(); - call_with( - &imported(py, TEMPLATING_MODULE, "expand_template_data")?, - &[ - ("value", connection), - ("variables", effective_vars.clone()), - ("context_values", PyDict::new(py).into_any()), - ( - "context_label", - PyString::new(py, CONNECTION_LABEL).into_any(), - ), - ("allow_context", falsy()), - ("preserve_context_tokens", falsy()), - ("preserve_unknown_context", falsy()), - ], + expanded_template_data( + &connection, + (effective_vars.downcast()?, &PyDict::new(py)), + ( + TemplateOptions { + allow_context: false, + preserve_context_tokens: false, + preserve_unknown_context: false, + }, + CONNECTION_LABEL, + ), ) } @@ -88,7 +91,101 @@ pub(crate) fn build_effective_vars<'py>( let py: Python<'py> = project_config.py(); let merged: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "effective_config_vars")? .call1(((project_config, local_config), target_config, cli_vars))?; - imported(py, TEMPLATING_MODULE, "expand_effective_vars")?.call1((merged,)) + expanded_effective_vars(merged.downcast()?) +} + +/// The selected target name of a discovered project and its effective config, if any. +fn target_selection<'py>( + discovered: &Bound<'py, PyAny>, + selected: Option>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = discovered.py(); + let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; + let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; + let selected: Bound<'py, PyAny> = selected.unwrap_or_else(|| py.None().into_bound(py)); + let Some(name) = resolve_target_name(py, &project_config, &local_config, &selected)? else { + return Ok((py.None().into_bound(py), py.None().into_bound(py))); + }; + let name: Bound<'py, PyAny> = name.into_any(); + let config: Bound<'py, PyAny> = + resolve_target_config(py, &project_config, &local_config, &name)?; + Ok((name, config)) +} + +/// The effective vars of a discovered project for `target_config` and optional CLI vars. +fn discovered_vars<'py>( + discovered: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, + cli_vars: Option>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let cli_vars: Bound<'py, PyAny> = match cli_vars { + Some(cli_vars) if !cli_vars.is_none() => cli_vars, + _ => PyDict::new(py).into_any(), + }; + build_effective_vars( + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, + target_config, + &cli_vars, + ) +} + +/// Build effective target config without compiling resources. +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, selected_target = None))] +pub(crate) fn build_effective_target_config<'py>( + discovered_inputs: &Bound<'py, PyAny>, + selected_target: Option>, +) -> PyResult> { + Ok(target_selection(discovered_inputs, selected_target)?.1) +} + +/// Build the effective project connection config without compiling resources. +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, selected_target = None, cli_vars = None))] +pub(crate) fn build_effective_connection_config<'py>( + discovered_inputs: &Bound<'py, PyAny>, + selected_target: Option>, + cli_vars: Option>, +) -> PyResult> { + let (_, target_config) = target_selection(discovered_inputs, selected_target)?; + let effective_vars: Bound<'py, PyAny> = + discovered_vars(discovered_inputs, &target_config, cli_vars)?; + build_effective_connection( + &discovered_inputs.getattr("project_config")?, + &discovered_inputs.getattr("local_config")?, + &target_config, + &effective_vars, + ) +} + +/// Build environment, vars and invocation id without compiling resources. +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, selected_target = None, cli_vars = None))] +pub(crate) fn build_effective_runtime_config<'py>( + discovered_inputs: &Bound<'py, PyAny>, + selected_target: Option>, + cli_vars: Option>, +) -> PyResult> { + let py: Python<'py> = discovered_inputs.py(); + let (target_name, target_config) = target_selection(discovered_inputs, selected_target)?; + let effective_vars: Bound<'py, PyAny> = + discovered_vars(discovered_inputs, &target_config, cli_vars)?; + let run_id: Bound<'py, PyAny> = run_id(&py.None().into_bound(py))?; + PyTuple::new(py, [target_name, effective_vars, run_id]) +} + +/// Build effective settings without compiling resources. +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs))] +pub(crate) fn build_effective_settings_config<'py>( + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + build_effective_settings( + &discovered_inputs.getattr("project_config")?, + &discovered_inputs.getattr("local_config")?, + ) } /// `build_effective_vars` for a discovered project and target, without CLI vars. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs index 9225615d45..5737802a12 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs @@ -4,5 +4,6 @@ pub(crate) mod effective_config; pub(crate) mod model_config; pub(crate) mod model_inputs; pub(crate) mod registration; +pub(crate) mod sql_expansion_context; mod storage_policies; pub(crate) mod target; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index e208d7f7bb..24d16e9b97 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -2,13 +2,16 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; -use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; +use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple}; +use sqlbuild_analysis::assembly::project::main::sql_analysis_opt_out_help::sql_analysis_opt_out_help; +use crate::bindings::_helpers::attachments::cursor_intrinsics::reserved_cursor_markers; use crate::bindings::_helpers::compile_attachment::model_config::native_model_config_session; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::compiled_project::loop_classes::model_loop_classes; +use crate::bindings::_helpers::render::declaration_values::constant_declaration_sql; use crate::bindings::_helpers::render::macro_expansion_result::macro_expansion_classes; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -19,11 +22,7 @@ const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; -const VALIDATION_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.validation"; const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; -const CURSOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; -const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; -const TEMPLATING_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.templating"; const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; const PROJECT_VARS_MODULE: &str = "sqlbuild._native"; const MACRO_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; @@ -43,11 +42,9 @@ fn dict<'py>( Ok(built) } -/// `sql_analysis_opt_out_help(model_owned=...)`. -fn opt_out_help<'py>(py: Python<'py>, model_owned: bool) -> PyResult> { - let keywords: Bound<'py, PyDict> = PyDict::new(py); - keywords.set_item("model_owned", model_owned)?; - imported(py, VALIDATION_MODULE, "sql_analysis_opt_out_help")?.call((), Some(&keywords)) +/// The parser opt-out help as a Python string. +fn opt_out_help(py: Python<'_>, model_owned: bool) -> PyResult> { + Ok(PyString::new(py, &sql_analysis_opt_out_help(model_owned)).into_any()) } /// The host the loop calls back into: `constant_text(*, declaration, file_path)`. @@ -61,19 +58,13 @@ fn loop_host<'py>(context: &Bound<'py, PyAny>) -> PyResult> { let context: &Bound<'_, PyAny> = context.bind(py); let passed: Bound<'_, PyDict> = keywords.map_or_else(|| PyDict::new(py), |keywords| keywords.clone()); - call_with( - &imported(py, DECLARATIONS_MODULE, "render_constant_declaration")?, - &[ - ("declaration", passed.as_any().get_item("declaration")?), - ("value_renderer", context.getattr("value_renderer")?), - ( - "collection_rendering", - context.getattr("collection_rendering")?, - ), - ("file_path", passed.as_any().get_item("file_path")?), - ], - ) - .map(Bound::unbind) + let file_path: Bound<'_, PyAny> = passed.as_any().get_item("file_path")?; + let rendered: String = constant_declaration_sql( + &passed.as_any().get_item("declaration")?, + &context.getattr("value_renderer")?, + (&context.getattr("collection_rendering")?, Some(&file_path)), + )?; + Ok(PyString::new(py, &rendered).into_any().unbind()) }; let keywords: Bound<'py, PyDict> = PyDict::new(py); keywords.set_item( @@ -252,7 +243,7 @@ pub(crate) fn build_model_inputs<'py>( ), ( "record_template_reads", - imported(py, TEMPLATING_MODULE, "record_template_reads")?, + imported(py, NATIVE_MODULE, "record_template_reads")?, ), ( "visible_declarations", @@ -291,7 +282,7 @@ pub(crate) fn build_model_inputs<'py>( classes, ), syntax.getattr("native_mapping")?, - imported(py, CURSOR_MODULE, "cursor_intrinsic_semantics")?.call0()?, + reserved_cursor_markers(), ( ( settings.getattr("sql_analysis")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs index a5a741456a..ea6f752684 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs @@ -4,10 +4,15 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; use crate::bindings::_helpers::compile_attachment::{ - cursor_roles, declaration_scope, effective_config, model_config, model_inputs, target, + cursor_roles, declaration_scope, effective_config, model_config, model_inputs, + sql_expansion_context, target, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + sql_expansion_context::build_sql_expansion_context, + module + )?)?; module.add_function(wrap_pyfunction!( effective_config::build_effective_connection, module @@ -20,6 +25,22 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { effective_config::build_effective_vars, module )?)?; + module.add_function(wrap_pyfunction!( + effective_config::build_effective_target_config, + module + )?)?; + module.add_function(wrap_pyfunction!( + effective_config::build_effective_connection_config, + module + )?)?; + module.add_function(wrap_pyfunction!( + effective_config::build_effective_runtime_config, + module + )?)?; + module.add_function(wrap_pyfunction!( + effective_config::build_effective_settings_config, + module + )?)?; module.add_function(wrap_pyfunction!( model_config::native_model_config_session, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs new file mode 100644 index 0000000000..5bed96ebad --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs @@ -0,0 +1,195 @@ +//! The SQL expansion inputs lint uses to expand authored SQL outside a compile. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const USER_MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; + +/// Build the declaration scope of `discovered`; a static scope lends only its index to macros. +fn declaration_scope<'py>( + discovered: &Bound<'py, PyAny>, + static_scope: Bound<'py, PyAny>, + sql_lexical_syntax: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let loaded_macros: Bound<'py, PyAny> = imported(py, USER_MACROS_MODULE, "load_project_macros")? + .call1((discovered.getattr("macro_files")?,))?; + if !static_scope.is_none() { + let rebound: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "rebind_declaration_scope")?, + &[ + ("scope", static_scope), + ("discovered_inputs", discovered.clone()), + ("loaded_macros", loaded_macros.clone()), + ], + )?; + if !rebound.is_none() { + return Ok(rebound); + } + } + call_with( + &imported(py, NATIVE_MODULE, "build_declaration_scope")?, + &[ + ("discovered_inputs", discovered.clone()), + ("loaded_macros", loaded_macros), + ("sql_lexical_syntax", sql_lexical_syntax), + ], + ) +} + +/// Each model file's own enum and constant declarations, by file path. +fn local_declarations<'py>(discovered: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = discovered.py(); + let context: Bound<'py, PyAny> = + imported(py, COMPILE_MODELS_MODULE, "DeclarationResolutionContext")?; + let by_name = |declarations: Bound<'py, PyAny>| -> PyResult> { + let named: Bound<'py, PyDict> = PyDict::new(py); + for declaration in declarations.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + named.set_item(declaration.getattr("name")?, declaration)?; + } + Ok(named.into_any()) + }; + let local: Bound<'py, PyDict> = PyDict::new(py); + for model_file in discovered.getattr("model_files")?.try_iter()? { + let model_file: Bound<'py, PyAny> = model_file?; + local.set_item( + model_file.getattr("file_path")?, + call_with( + &context, + &[ + ("enums", by_name(model_file.getattr("enum_declarations")?)?), + ( + "constants", + by_name(model_file.getattr("constant_declarations")?)?, + ), + ], + )?, + )?; + } + Ok(local) +} + +/// Assemble SQL expansion inputs; a static scope lends only its index to private macros. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_sql_expansion_context<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_sql_expansion_context", + options, + ( + &["project_dir", "value_renderer"], + &[ + "cli_vars", + "discovered_inputs", + "declaration_scope", + "static_declaration_scope", + ], + ), + )?; + let value_renderer: Bound<'py, PyAny> = arguments.value("value_renderer")?; + let mut discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; + if discovered.is_none() { + discovered = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[ + ("project_dir", arguments.value("project_dir")?), + ( + "sql_analysis_enabled_override", + PyBool::new(py, false).to_owned().into_any(), + ), + ], + )?; + } + let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; + let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; + let mut cli_vars: Bound<'py, PyAny> = arguments.value("cli_vars")?; + if cli_vars.is_none() { + cli_vars = PyDict::new(py).into_any(); + } + let effective_vars: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "build_effective_vars")?, + &[ + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ("target_config", py.None().into_bound(py)), + ("cli_vars", cli_vars), + ], + )?; + let mut scope: Bound<'py, PyAny> = arguments.value("declaration_scope")?; + if scope.is_none() { + scope = declaration_scope( + &discovered, + arguments.value("static_declaration_scope")?, + value_renderer.getattr("sql_lexical_syntax")?, + )?; + } + let (enums, constants): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + imported(py, NATIVE_MODULE, "public_declaration_indexes")? + .call1((&discovered,))? + .extract()?; + let collection_rendering: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_effective_collection_rendering")?, + &[ + ("project_config", project_config.clone()), + ("declaration_override", py.None().into_bound(py)), + ], + )?; + let macro_context: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "MacroContext")?, + &[ + ( + "adapter_name", + call_with( + &imported(py, NATIVE_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", project_config.clone()), + ("local_config", local_config), + ], + )?, + ), + ( + "sql_analysis_enabled", + PyBool::new(py, false).to_owned().into_any(), + ), + ("target_name", project_config.getattr("default_target")?), + ("vars", effective_vars.clone()), + ("_value_renderer", value_renderer.clone()), + ("_collection_rendering", collection_rendering.clone()), + ( + "_enforce_explicit_references", + project_config + .getattr("references")? + .getattr("enforce_explicit")?, + ), + ], + )?; + call_with( + &imported(py, COMPILE_MODELS_MODULE, "SqlExpansionContext")?, + &[ + ("effective_vars", effective_vars), + ("loaded_macros", scope.getattr("loaded_macros")?), + ("macro_context", macro_context), + ("enums", enums), + ("constants", constants), + ("value_renderer", value_renderer), + ("collection_rendering", collection_rendering), + ( + "local_declarations", + local_declarations(&discovered)?.into_any(), + ), + ("declaration_resolver", scope.getattr("resolver")?), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs index cabc68551f..23fc340a8d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -8,14 +8,13 @@ use crate::bindings::_helpers::compile_attachment::effective_config::project_var use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::model_config::template_data::{early_model_templates, run_id}; -const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; -const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const TARGET_NAME_MODULE: &str = "sqlbuild._native"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const FRONTIER_CONSTANTS_MODULE: &str = "sqlbuild.compiler.frontier.constants"; const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; -const CONTEXT_TEMPLATES_MODULE: &str = - "sqlbuild.compiler.compile._helpers.render.context_templates"; const DATABASE_KEY: &str = "database"; const SCHEMA_KEY: &str = "schema"; @@ -217,24 +216,10 @@ pub(crate) fn build_effective_target_namespace<'py>( SCHEMA_KEY, first_truthy([target_value(SCHEMA_KEY)?, connection_schema, default_schema])?, )?; - let run_id: Bound<'py, PyAny> = call_with( - &imported(py, CONTEXT_TEMPLATES_MODULE, "resolve_run_id")?, - &[("selected_run_id", py.None().into_bound(py))], - )?; - let resolved: Bound<'py, PyDict> = call_with( - &imported( - py, - CONTEXT_TEMPLATES_MODULE, - "resolve_early_model_templates", - )?, - &[ - ("values", values.into_any()), - ("effective_vars", effective_vars), - ("effective_target_name", target_name.clone()), - ("run_id", run_id), - ], - )? - .downcast_into()?; + let run_id: Bound<'py, PyAny> = run_id(&py.None().into_bound(py))?; + let resolved: Bound<'py, PyDict> = + early_model_templates(values.as_any(), &effective_vars, (&target_name, &run_id))? + .downcast_into()?; PyTuple::new( py, [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index c7b0e058e3..18eeddf21a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -19,8 +19,8 @@ use crate::bindings::_helpers::compile_outputs::json_report::{ emit_json_report, emit_orjson_report, }; use crate::bindings::_helpers::discovery::hook_entries::hook_entries; +use crate::bindings::_helpers::semantic_shapes::coverage::semantic_coverage; -const SEMANTIC_COVERAGE_MODULE: &str = "sqlbuild.compiler.compile.main._semantic_coverage"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; @@ -163,10 +163,7 @@ impl<'py> ReportInputs<'py> { if !self.sql_validation_enabled { return Ok(PyDict::new(py)); } - let reasons: Bound<'py, PyDict> = - imported(py, SEMANTIC_COVERAGE_MODULE, "get_semantic_coverage")? - .call1((&self.project,))? - .downcast_into()?; + let reasons: Bound<'py, PyDict> = semantic_coverage(&self.project)?; let Some(names) = self.selected_names(MODEL_TYPE)? else { return Ok(reasons); }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index ff64912957..31a965b2b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const DISCOVER_MODULE: &str = "sqlbuild._native"; const REJECT_SELECTORS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const SELECTED_GRAPH_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index d8d224d721..d88b9f44d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -13,8 +13,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; -pub(crate) const NATIVE_STAGES_MODULE: &str = - "sqlbuild.compiler.compile._helpers.native_stages.assembly"; const ANALYSIS_CACHE_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.cache"; pub(crate) const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._record"; @@ -274,7 +272,7 @@ fn record_expression_shapes<'py>( } } let shapes: Vec> = call_with( - &imported(py, NATIVE_STAGES_MODULE, "expression_source_shapes")?, + &imported(py, NATIVE_MODULE, "infer_native_expression_source_shapes")?, &[ ("expressions", PyTuple::new(py, expressions)?.into_any()), ("profile", context.profile.clone()), @@ -386,7 +384,10 @@ fn model_analyses<'py>( ), ], )?; - imported(py, NATIVE_STAGES_MODULE, "analyze_model_sql")?.call1((analysis_request,)) + call_with( + &imported(py, NATIVE_MODULE, "analyze_native_model_sql")?, + &[("request", analysis_request)], + ) }) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs index 87816897fb..65482d6dc3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs @@ -3,15 +3,14 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; +use crate::bindings::_helpers::attachments::cursor_intrinsics::validated_model_cursor_sql; use crate::bindings::_helpers::compile_driver::assembly_project::ADAPTER_NAME_MODULE; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::render::declaration_values::declaration_scope_resolver; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const CURSOR_INTRINSICS_MODULE: &str = - "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; /// The `CompiledSqlScenario` fields copied from its input, in Python's keyword order. const SCENARIO_FIELDS: [&str; 12] = [ "sql_body", @@ -64,21 +63,12 @@ pub(crate) fn test_model_query_overrides<'py>( declaration_scope.getattr("resolver")?.getattr("lookup")? }; let loaded_macros: Bound<'py, PyAny> = inputs.getattr("loaded_macros")?; - let declaration_resolver: Bound<'py, PyAny> = call_with( - &imported(py, DECLARATIONS_MODULE, "build_declaration_scope_resolver")?, - &[ - ("discovered_inputs", inputs.getattr("discovered_inputs")?), - ("scope_index", inputs.getattr("scope_index")?), - ("loaded_macros", loaded_macros.clone()), - ("lookup", lookup), - ], + let declaration_resolver: Bound<'py, PyAny> = declaration_scope_resolver( + &inputs.getattr("discovered_inputs")?, + &inputs.getattr("scope_index")?, + (&loaded_macros, &lookup), )?; let expand: Bound<'py, PyAny> = py.import(NATIVE_MODULE)?.getattr("expand_sql_macros")?; - let intrinsics: Bound<'py, PyAny> = imported( - py, - CURSOR_INTRINSICS_MODULE, - "get_validated_model_cursor_intrinsics", - )?; for model_input in inputs.getattr("model_inputs")?.try_iter()? { let model_input: Bound<'py, PyAny> = model_input?; let file_path: Bound<'py, PyAny> = @@ -102,16 +92,13 @@ pub(crate) fn test_model_query_overrides<'py>( ], )? .getattr("sql")?; - let query: Bound<'py, PyAny> = call_with( - &intrinsics, - &[ - ("sql", expanded), - ( - "config_values", - model_input.getattr("config")?.getattr("values")?, - ), - ("model_name", model_name.clone()), - ], + let query: String = validated_model_cursor_sql( + py, + &expanded.extract::()?, + ( + &model_name.extract::()?, + &model_input.getattr("config")?.getattr("values")?, + ), )?; overrides.set_item(model_name, query)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs new file mode 100644 index 0000000000..a50bdb50a0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs @@ -0,0 +1,134 @@ +//! Every model's native facts and the project's resource facts, from the native stages. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::assembly_analysis::AnalysisContext; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const PROJECT_ASSEMBLY_MODELS_MODULE: &str = "sqlbuild.compiler.project_assembly.models"; + +/// `NativeProjectFacts` of `inputs`: resources, syntax validity, deps and standalone pivot proofs. +pub(crate) fn native_project_facts<'py>( + inputs: &Bound<'py, PyAny>, + context: &AnalysisContext<'py>, + names: &Bound<'py, PyAny>, + session: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let model_inputs: Bound<'py, PyAny> = inputs.getattr("model_inputs")?; + let dialect: Bound<'py, PyAny> = context.profile.getattr("sql_analysis_dialect")?; + let analysed: bool = !context.analyses.is_none(); + let succeeded: Bound<'py, PyAny> = if analysed { + context.analyses.getattr("succeeded")? + } else { + PyFrozenSet::empty(py)?.into_any() + }; + let positions: Bound<'py, PyAny> = if analysed { + context.analyses.getattr("positions")? + } else { + PyDict::new(py).into_any() + }; + let syntax_checks: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "model_syntax_checks")?, + &[ + ("model_inputs", model_inputs.clone()), + ("analysis_model_names", names.clone()), + ("analysis_succeeded", succeeded), + ], + )?; + let resources: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "assemble_native_project_resources")?, + &[ + ("inputs", inputs.clone()), + ("dialect", dialect.clone()), + ("syntax_checks", syntax_checks), + ], + )?; + let proofs: Bound<'py, PyDict> = pivot_proofs( + &call_with( + &imported(py, NATIVE_MODULE, "standalone_pivot_models")?, + &[ + ("model_inputs", model_inputs.clone()), + ("analysed", positions), + ], + )? + .cast_into::()?, + context, + session, + )?; + let model_facts: Bound<'py, PyAny> = + imported(py, PROJECT_ASSEMBLY_MODELS_MODULE, "NativeModelFacts")?; + let deps: Bound<'py, PyAny> = resources.getattr("model_deps")?; + let syntax_valid: Bound<'py, PyAny> = resources.getattr("model_syntax_valid")?; + let mut models: Vec> = Vec::new(); + for index in 0..model_inputs.len()? { + let proof: Bound<'py, PyAny> = match proofs.get_item(index)? { + Some(proof) => proof, + None => py.None().into_bound(py), + }; + models.push(call_with( + &model_facts, + &[ + ("deps", deps.get_item(index)?), + ("dynamic_contract", proof), + ("syntax_valid", syntax_valid.get_item(index)?), + ], + )?); + } + call_with( + &imported(py, PROJECT_ASSEMBLY_MODELS_MODULE, "NativeProjectFacts")?, + &[ + ("models", PyTuple::new(py, models)?.into_any()), + ("resources", resources), + ], + ) +} + +/// Each standalone pivot model's dynamic contract proof, by model position. +fn pivot_proofs<'py>( + pivots: &Bound<'py, PyDict>, + context: &AnalysisContext<'py>, + session: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = pivots.py(); + let tables: Bound<'py, PyAny> = call_with( + &imported(py, ANALYSIS_MODELS_MODULE, "NativePivotTables")?, + &[ + ("dialect", context.profile.getattr("sql_analysis_dialect")?), + ("column_types_by_table", context.column_types.clone()), + ( + "authoritative_column_types_by_table", + context.complete_schemas.clone(), + ), + ("column_nullability_by_table", context.nullability.clone()), + ( + "dynamic_families_by_table", + context.dynamic_families.clone(), + ), + ], + )?; + let proven: Vec> = call_with( + &imported(py, NATIVE_MODULE, "prove_native_dynamic_contracts")?, + &[ + ("session", session.clone()), + ("tables", tables), + ("models", PyTuple::new(py, pivots.values())?.into_any()), + ], + )? + .try_iter()? + .collect::>()?; + if proven.len() != pivots.len() { + return Err(PyValueError::new_err( + "native pivot proofs do not match the standalone pivot models", + )); + } + let proofs: Bound<'py, PyDict> = PyDict::new(py); + for (index, proof) in pivots.keys().iter().zip(proven) { + proofs.set_item(index, proof)?; + } + Ok(proofs) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index 0e23885df8..321999347e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -3,6 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; +use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::compile_driver::assembly_analysis::{ ANALYSIS_MODELS_MODULE, AnalysisContext, model_stem, selected_for_analysis, @@ -12,15 +13,12 @@ use crate::bindings::_helpers::hand_built::compiled_model::compiled_model as han use crate::bindings::_helpers::render::macro_calls::top_level_macro_call_names; const NATIVE_MODULE: &str = "sqlbuild._native"; -const CURSOR_INTRINSICS_MODULE: &str = - "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; /// The Python stages each model's assembly calls, looked up once per project. struct ModelStages<'py> { - analysis_sql: Bound<'py, PyAny>, placeholders: Bound<'py, PyAny>, validate_hooks: Bound<'py, PyAny>, validate_sql: Bound<'py, PyAny>, @@ -44,11 +42,6 @@ pub(crate) fn compiled_models<'py>( let py: Python<'py> = inputs.py(); let native: Bound<'py, PyAny> = py.import(NATIVE_MODULE)?.into_any(); let stages: ModelStages<'py> = ModelStages { - analysis_sql: imported( - py, - CURSOR_INTRINSICS_MODULE, - "cursor_intrinsics_analysis_sql", - )?, placeholders: native.getattr("model_placeholders")?, validate_hooks: native.getattr("validate_hook_sql_syntax")?, validate_sql: native.getattr("validate_sql_syntax")?, @@ -83,16 +76,15 @@ fn compiled_model<'py>( let model_name: Bound<'py, PyAny> = model_stem(model_input)?; let syntax_validated: bool = request.native_model.getattr("syntax_valid")?.is_truthy()?; let config_values: Bound<'py, PyAny> = model_input.getattr("config")?.getattr("values")?; - let analysis_query_sql: Bound<'py, PyAny> = call_with( - &stages.analysis_sql, - &[ - ("sql", model_input.getattr("query_sql")?), - ( - "cursor_type", - config_values.call_method1("get", ("cursor_type",))?, - ), - ], - )?; + let analysis_query_sql: Bound<'py, PyAny> = PyString::new( + py, + &analysis_cursor_sql( + py, + &model_input.getattr("query_sql")?.extract::()?, + &config_values.call_method1("get", ("cursor_type",))?, + )?, + ) + .into_any(); let placeholders: Bound<'py, PyAny> = stages.placeholders.call1((model_input,))?; let input_validated: bool = model_input.getattr("sql_validation_enabled")?.is_truthy()?; let location: Bound<'py, PyTuple> = PyTuple::new( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index 5f71574f50..28e30a54b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -1,34 +1,29 @@ //! Compiled project assembly: the project's models, resources, checks and semantic diagnostics. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTuple}; use sqlbuild_analysis::assembly::project::main::configured_namespace::configured_namespace; use crate::bindings::_helpers::compile_driver::assembly::AssemblyRequest; use crate::bindings::_helpers::compile_driver::assembly_analysis::{ - AnalysisContext, NATIVE_STAGES_MODULE, analysis_context, and_not, -}; -use crate::bindings::_helpers::compile_driver::assembly_checks::{ - compiled_sql_scenarios, test_model_query_overrides, + AnalysisContext, analysis_context, and_not, }; +use crate::bindings::_helpers::compile_driver::assembly_checks::compiled_sql_scenarios; +use crate::bindings::_helpers::compile_driver::assembly_facts::native_project_facts; use crate::bindings::_helpers::compile_driver::assembly_models::{ binding_rows, compiled_models, project_binding_diagnostics, }; +use crate::bindings::_helpers::compile_driver::assembly_sql_tests::assembled_sql_tests; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::model_config::template_data::early_model_templates; use crate::bindings::_helpers::scopes::scope_errors::report_scope_index_errors; +use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; const NATIVE_MODULE: &str = "sqlbuild._native"; -const SCOPE_USAGES_MODULE: &str = "sqlbuild.compiler.compile.main._scope_index_with_compile_usages"; -const CONTEXT_TEMPLATES_MODULE: &str = - "sqlbuild.compiler.compile._helpers.render.context_templates"; -pub(crate) const ADAPTER_NAME_MODULE: &str = - "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +pub(crate) const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTER_TYPES_MODULE: &str = "sqlbuild.adapter.contract.types"; -const SCENARIO_CONFIG_MODULE: &str = - "sqlbuild.spec.contracts.main.resolve_effective_scenario_config"; -const SQL_TESTS_MODULE: &str = "sqlbuild.compiler.compile._helpers.native_stages.sql_tests"; -const RECOVERY_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.recovery"; +const SCENARIO_CONFIG_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const SOURCE_SLOT: &str = "source"; const SEED_SLOT: &str = "seed"; @@ -85,31 +80,9 @@ pub(crate) fn assembled_project<'py>( } else { context.analyses.getattr("session")? }; - let native: Bound<'py, PyAny> = call_with( - &imported(py, NATIVE_STAGES_MODULE, "native_project_facts")?, - &[ - ("inputs", inputs.clone()), - ("dialect", context.profile.getattr("sql_analysis_dialect")?), - ("analysis_model_names", names.clone()), - ("analyses", context.analyses.clone()), - ("session", native_session.clone()), - ("column_types_by_table", context.column_types.clone()), - ( - "authoritative_column_types_by_table", - context.complete_schemas.clone(), - ), - ("column_nullability_by_table", context.nullability.clone()), - ( - "dynamic_families_by_table", - context.dynamic_families.clone(), - ), - ], - )?; + let native: Bound<'py, PyAny> = native_project_facts(inputs, &context, names, &native_session)?; let resources: Bound<'py, PyAny> = native.getattr("resources")?; - let scope_index: Bound<'py, PyAny> = call_with( - &imported(py, SCOPE_USAGES_MODULE, "scope_index_with_compile_usages")?, - &[("inputs", inputs.clone())], - )?; + let scope_index: Bound<'py, PyAny> = scope_index_with_compile_usages(inputs)?; report_scope_index_errors(&scope_index)?; let target_values: Bound<'py, PyAny> = effective_target_values(inputs)?; let models: Bound<'py, PyTuple> = @@ -123,7 +96,7 @@ pub(crate) fn assembled_project<'py>( let project: Bound<'py, PyAny> = project_view(inputs, &fields, &models)?; let binding_results: Bound<'py, PyDict> = binding_rows(&context.analyses)?; call_with( - &imported(py, RECOVERY_MODULE, "complete_semantic_diagnostics")?, + &imported(py, NATIVE_MODULE, "complete_semantic_project")?, &[ ("project", project), ("profile", context.profile.clone()), @@ -134,7 +107,7 @@ pub(crate) fn assembled_project<'py>( &inputs.getattr("no_sql_validation")?, )?, ), - ("native_session", native_session), + ("session", native_session), ("binding_results", binding_results.into_any()), ], ) @@ -176,24 +149,13 @@ fn effective_target_values<'py>(inputs: &Bound<'py, PyAny>) -> PyResult( ] { fields.set_item(name, compiled_resources(inputs, resources, kind)?)?; } - let inputs_owned: pyo3::Py = inputs.clone().unbind(); - let overrides: Bound<'py, PyCFunction> = PyCFunction::new_closure( - py, - None, - None, - move |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| { - let py: Python<'_> = arguments.py(); - test_model_query_overrides(&arguments.get_item(0)?, inputs_owned.bind(py)) - .map(|overrides| overrides.into_any().unbind()) - }, - )?; - fields.set_item( - "sql_tests", - call_with( - &imported(py, SQL_TESTS_MODULE, "assemble_sql_tests")?, - &[ - ("inputs", inputs.clone()), - ("macro_mock_queries", overrides.into_any()), - ], - )?, - )?; + fields.set_item("sql_tests", assembled_sql_tests(inputs)?)?; fields.set_item("sql_scenarios", compiled_sql_scenarios(inputs)?)?; for name in ["loader_functions", "hook_functions"] { fields.set_item(name, discovered.getattr(name)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs new file mode 100644 index 0000000000..40601cab98 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs @@ -0,0 +1,117 @@ +//! Every SQL test assembled natively; macro-mocked tests' model queries run the user's macros. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyList, PyTuple}; + +use crate::bindings::_helpers::boundary::panics::NativeCompilerError; +use crate::bindings::_helpers::compile_driver::assembly_checks::test_model_query_overrides; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const DECIMAL_MODULE: &str = "decimal"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const SQL_TEST_INPUT_FAILURE: &str = "input"; +const SQL_TEST_DECIMAL_OVERFLOW: &str = "decimal_overflow"; + +/// Every compiled SQL test, raising each assembly failure in the tests' order. +pub(crate) fn assembled_sql_tests<'py>( + inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let assemblies: Vec> = call_with( + &imported(py, NATIVE_MODULE, "assemble_native_sql_tests")?, + &[("inputs", inputs.clone())], + )? + .try_iter()? + .collect::>()?; + let test_inputs: Vec> = inputs + .getattr("test_inputs")? + .try_iter()? + .collect::>()?; + if assemblies.len() != test_inputs.len() { + return Err(NativeCompilerError::new_err( + "native SQL test assembly returned a different number of tests", + )); + } + let report: Bound<'py, PyAny> = imported(py, COLLECTOR_MODULE, "report_compile_diagnostic")?; + let mocked_input: Bound<'py, PyAny> = + imported(py, COMPILE_MODELS_MODULE, "CompileModelSqlTestInputPayload")?; + let mut tests: Vec> = Vec::with_capacity(assemblies.len()); + for (test_input, assembly) in test_inputs.iter().zip(assemblies) { + let failure: Bound<'py, PyAny> = assembly.getattr("failure")?; + let kind: Option = if failure.is_none() { + None + } else { + Some(failure.getattr("kind")?.extract()?) + }; + let mut test: Bound<'py, PyAny> = assembly.getattr("test")?; + if test.is_none() { + return Err(assembly_error(py, &failure, kind.as_deref())?); + } + for item in assembly.getattr("diagnostics")?.try_iter()? { + let (key, diagnostic): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + call_with(&report, &[("key", key), ("diagnostic", diagnostic)])?; + } + let payload: Bound<'py, PyAny> = test_input.getattr("payload")?; + if payload.is_instance(&mocked_input)? && payload.getattr("macro_mocks")?.is_truthy()? { + test = with_model_queries( + &test, + test_model_query_overrides(test_input, inputs)?.as_any(), + )?; + } + if kind.as_deref() == Some(SQL_TEST_DECIMAL_OVERFLOW) { + let overflow: Bound<'py, PyAny> = imported(py, DECIMAL_MODULE, "Overflow")?; + let raised: Bound<'py, PyAny> = overflow.call1((PyList::new(py, [&overflow])?,))?; + return Err(PyErr::from_value(raised)); + } + tests.push(test); + } + PyTuple::new(py, tests) +} + +/// The failure of a test that did not assemble: input errors as `CompileInputError`. +fn assembly_error( + py: Python<'_>, + failure: &Bound<'_, PyAny>, + kind: Option<&str>, +) -> PyResult { + let message: String = if failure.is_none() { + String::new() + } else { + failure.getattr("message")?.extract()? + }; + if kind == Some(SQL_TEST_INPUT_FAILURE) { + let error: Bound<'_, PyAny> = + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?.call1((message,))?; + return Ok(PyErr::from_value(error)); + } + Ok(NativeCompilerError::new_err(message)) +} + +/// `test` with its model payload's query overrides set to `queries`. +fn with_model_queries<'py>( + test: &Bound<'py, PyAny>, + queries: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = test.py(); + let payload: Bound<'py, PyAny> = test.getattr("payload")?; + if !payload.is_instance(&imported( + py, + COMPILE_MODELS_MODULE, + "CompiledModelSqlTestPayload", + )?)? { + return Ok(test.clone()); + } + let replace: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?; + let payload: Bound<'py, PyAny> = replace.call( + (payload,), + Some(&keywords( + py, + &[("model_query_overrides", queries.clone())], + )?), + )?; + replace.call((test,), Some(&keywords(py, &[("payload", payload)])?)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index f59594105c..d0d189dce9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -4,6 +4,7 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyBool, PyDict, PyDictMethods, PyMapping, PyString, PyStringMethods, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::compile_command::report::{ format_compile_json, format_compile_text, json_semantic_notice, }; @@ -22,7 +23,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const REUSE_MODELS_MODULE: &str = "sqlbuild.cli.compile_reuse.models"; const REUSE_TYPES_MODULE: &str = "sqlbuild.cli.compile_reuse.types"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; @@ -128,16 +128,7 @@ fn compile_status<'py>(request: &Bound<'py, PyAny>) -> PyResult( let discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; let adapter_context: Bound<'py, PyAny> = arguments.value("adapter_context")?; let target: TargetContext<'py> = target_context(arguments, &discovered, &adapter_context)?; - let run_id: Bound<'py, PyAny> = call_with( - &imported(py, CONTEXT_TEMPLATES_MODULE, "resolve_run_id")?, - &[("selected_run_id", arguments.value("run_id")?)], - )?; + let run_id: Bound<'py, PyAny> = run_id(&arguments.value("run_id")?)?; let loaded_macros: Bound<'py, PyAny> = imported(py, MACROS_MODULE, "load_project_macros")? .call1((discovered.getattr("macro_files")?,))?; let syntax: Bound<'py, PyAny> = adapter_context.getattr("sql_lexical_syntax")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index 7ebf3fd7ae..102f4a0e04 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -2,8 +2,10 @@ pub(crate) mod analysis; pub(crate) mod assembly; pub(crate) mod assembly_analysis; pub(crate) mod assembly_checks; +pub(crate) mod assembly_facts; pub(crate) mod assembly_models; pub(crate) mod assembly_project; +pub(crate) mod assembly_sql_tests; pub(crate) mod command; pub(crate) mod config_validation; pub(crate) mod deferral; @@ -19,5 +21,6 @@ pub(crate) mod python_calls; pub(crate) mod python_plan_entries; pub(crate) mod registration; pub(crate) mod relation_targets; +pub(crate) mod selection_phases; pub(crate) mod target_defaults; pub(crate) mod targets; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index ce8694d646..5f8f1e40c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -12,8 +12,8 @@ use crate::bindings::_helpers::compile_command::target_writer::{ use crate::bindings::_helpers::compile_driver::python_calls::{Status, call_with, imported}; use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; -const MANIFEST_MODULE: &str = "sqlbuild.compiler.manifest.main._build_graph_manifest"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const PYTHON_NODES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild.cli.output.models"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; @@ -68,7 +68,7 @@ pub(crate) fn manifest_payload<'py>( ], )?; let payload: Bound<'py, PyAny> = call_with( - &imported(py, MANIFEST_MODULE, "build_graph_manifest")?, + &imported(py, NATIVE_MODULE, "build_manifest")?, &[ ("project", graph.getattr("project")?), ("plan_output", py.None().into_bound(py)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index ffb6416eb8..fbe1f63570 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -11,12 +11,13 @@ use crate::bindings::_helpers::compile_driver::pipeline_entries::{ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; +use crate::bindings::_helpers::frontier::stage_capture::write_plan_capture; const NATIVE_MODULE: &str = "sqlbuild._native"; -const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; +const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild._native"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.analysis_selection"; +const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const TIMING_TRACKER_MODULE: &str = "sqlbuild.diagnostics.classes.build_phase_timing_tracker"; const RULE_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; @@ -390,6 +391,7 @@ fn planned<'py>( tracker.setattr("planning_seconds", planning_seconds)?; } let result: Bound<'py, PyAny> = result?; + write_plan_capture(&result.getattr("plan_output")?)?; imported(py, "dataclasses", "replace")?.call( (result,), Some(&keywords( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 69219b03c4..103aac491a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -26,11 +26,10 @@ const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; const PROMOTION_MODULE: &str = "sqlbuild._native"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; -const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.analysis_selection"; +const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._record"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; -const COLLECTION_RENDERING_MODULE: &str = - "sqlbuild.spec.contracts.main.resolve_effective_collection_rendering"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; +const COLLECTION_RENDERING_MODULE: &str = "sqlbuild._native"; /// Optional keywords of the project graph builders. const GRAPH_OPTIONAL: [&str; 7] = [ "selected_target", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs index 8e3d4733bd..de3bb1e189 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs @@ -17,7 +17,7 @@ const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; const NODE_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; -const RUN_LIFECYCLE_MODULE: &str = "sqlbuild.compiler.python_nodes.main._run_lifecycle"; +const NATIVE_MODULE: &str = "sqlbuild._native"; const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.compiler.python_nodes.models"; const PYTHON_NODE_TYPES_MODULE: &str = "sqlbuild.compiler.python_nodes.types"; const REF_TYPES_MODULE: &str = "sqlbuild.python_nodes.types"; @@ -172,7 +172,7 @@ fn build_python_run_plan_outputs<'py>( ], )?; let lifecycle: Bound<'py, PyAny> = call_with( - &imported(py, RUN_LIFECYCLE_MODULE, "build_python_sql_run_lifecycle")?, + &imported(py, NATIVE_MODULE, "build_python_sql_run_lifecycle")?, &[("selection", selection), ("python_graph", graph.clone())], )?; let entries: Bound<'py, PyTuple> = plan_entries( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs index 81700b0724..050f838115 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::compile_driver::{ analysis, assembly, command, deferral, diff, discovery, input_reads, inputs, pipeline, - pipeline_entries, project, python_plan_entries, relation_targets, targets, + pipeline_entries, project, python_plan_entries, relation_targets, selection_phases, targets, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -21,5 +21,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff::register(module)?; python_plan_entries::register(module)?; relation_targets::register(module)?; + selection_phases::register(module)?; targets::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs new file mode 100644 index 0000000000..29e2b31dd0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -0,0 +1,329 @@ +//! Unit-test selector splitting, analysis and rule selection, and static command compilation. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; +const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const SPLIT_SELECTORS_MODULE: &str = + "sqlbuild.compiler.planner.main.selection._split_unit_test_selectors"; +const PLANNER_SCOPES_MODULE: &str = + "sqlbuild.compiler.planner.main.selection._resolve_planner_scopes"; +const STATIC_SCOPE_MODULE: &str = "sqlbuild.compiler.planner.main.commands._scope"; +const STATIC_RELATIONS_MODULE: &str = "sqlbuild.compiler.planner.main.commands._relations"; +const DATACLASSES_MODULE: &str = "dataclasses"; + +/// Whether a selection names nothing to select or exclude. +fn unselective(selection: &Bound<'_, PyAny>) -> PyResult { + Ok(!selection.getattr("select")?.is_truthy()? && !selection.getattr("exclude")?.is_truthy()?) +} + +/// Options whose selectors name only resources, with unit tests selected apart. +fn unit_test_selectors_applied<'py>( + options: &Bound<'py, PyAny>, + discovered: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let resolved: Bound<'py, PyAny> = if options.is_none() { + imported(py, PIPELINE_MODELS_MODULE, "CompilePipelineOptions")?.call0()? + } else { + options.clone() + }; + if unselective(&resolved)? { + return Ok(resolved); + } + let split: Bound<'py, PyAny> = call_with( + &imported( + py, + SPLIT_SELECTORS_MODULE, + "split_project_unit_test_selectors", + )?, + &[ + ("select", resolved.getattr("select")?), + ("exclude", resolved.getattr("exclude")?), + ("discovered_inputs", discovered.clone()), + ( + "accepts_unit_tests", + resolved.getattr("accepts_unit_test_selectors")?, + ), + ], + )?; + imported(py, DATACLASSES_MODULE, "replace")?.call( + (resolved,), + Some(&keywords( + py, + &[ + ("select", split.getattr("select")?), + ("exclude", split.getattr("exclude")?), + ("sql_test_selection", split.getattr("sql_test_selection")?), + ], + )?), + ) +} + +/// The planner scopes `selection` resolves over `project`. +fn planner_scopes<'py>( + project: &Bound<'py, PyAny>, + selection: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + call_with( + &imported(py, PLANNER_SCOPES_MODULE, "resolve_planner_scopes")?, + &[ + ("project", project.clone()), + ( + "selection", + call_with( + &imported(py, PLANNER_MODELS_MODULE, "PlannerSelection")?, + &[ + ("select", selection.getattr("select")?), + ("exclude", selection.getattr("exclude")?), + ], + )?, + ), + ( + "policies", + call_with( + &imported(py, PLANNER_MODELS_MODULE, "PlannerPolicies")?, + &[("auto_load_sources", selection.getattr("auto_load_sources")?)], + )?, + ), + ], + ) +} + +/// The stale-warning scope's models of `scopes`. +fn scoped_models<'py>(scopes: &Bound<'py, PyAny>) -> PyResult> { + scopes + .getattr("stale_warning_scope")? + .getattr("models_by_name") +} + +/// Return options whose selectors name only resources, with unit tests selected apart. +#[pyfunction] +#[pyo3(signature = (*, options, discovered_inputs))] +fn apply_unit_test_selectors<'py>( + options: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + unit_test_selectors_applied(options, discovered_inputs) +} + +/// Align configured-rule subjects with models that received deep SQL analysis. +#[pyfunction] +#[pyo3(signature = (*, options, discovered_inputs, project))] +fn resolve_configured_rule_selection<'py>( + options: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = options.py(); + if !options + .getattr("sql_test_selection")? + .getattr("models_selected")? + .is_truthy()? + { + return Ok(PyFrozenSet::empty(py)?.into_any()); + } + let selection: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_compile_analysis_selection")?, + &[ + ("options", options.clone()), + ("discovered_inputs", discovered_inputs.clone()), + ], + )?; + if selection.is_none() || unselective(&selection)? { + return Ok(py.None().into_bound(py)); + } + let models: Bound<'py, PyAny> = scoped_models(&planner_scopes(project, &selection)?)?; + let keys: Vec> = models + .call_method0("values")? + .try_iter()? + .map(|model| model?.getattr("key")) + .collect::>()?; + Ok(PyFrozenSet::new(py, keys)?.into_any()) +} + +/// Return the models selected for deep analysis, or None to analyse every model. +#[pyfunction] +#[pyo3(signature = (*, compile_inputs, inference_profile, selection))] +fn resolve_analysis_model_names<'py>( + compile_inputs: &Bound<'py, PyAny>, + inference_profile: &Bound<'py, PyAny>, + selection: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = compile_inputs.py(); + if selection.is_none() || unselective(selection)? { + return Ok(py.None().into_bound(py)); + } + let structural_project: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "assemble_project")?, + &[ + ("inputs", compile_inputs.clone()), + ("inference_profile", inference_profile.clone()), + ( + "skip_column_inference", + pyo3::types::PyBool::new(py, true).to_owned().into_any(), + ), + ("analysis_model_names", PyFrozenSet::empty(py)?.into_any()), + ], + )?; + let models: Bound<'py, PyAny> = + scoped_models(&planner_scopes(&structural_project, selection)?)?; + Ok(PyFrozenSet::new(py, models.try_iter()?.collect::>>()?)?.into_any()) +} + +/// Locations of the deferred target's relations, when the options defer to one. +fn deferred_locations<'py>( + (discovered, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + options: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let defer_to: Bound<'py, PyAny> = options.getattr("defer_to")?; + if defer_to.is_none() { + return Ok(py.None().into_bound(py)); + } + let deferred_target: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_deferred_target_config")?, + &[ + ("discovered_inputs", discovered.clone()), + ("defer_to", defer_to), + ( + "current_target_name", + project.getattr("effective_target_name")?, + ), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "build_deferred_locations")?, + &[ + ("project", project.clone()), + ("deferred_target_config", deferred_target), + ("effective_vars", project.getattr("effective_vars")?), + ("default_schema", adapter.call_method0("default_schema")?), + ( + "default_database", + adapter.call_method0("default_database")?, + ), + ( + "render_qualified_name", + adapter.getattr("render_qualified_name")?, + ), + ], + ) +} + +/// Run canonical compile, selector, and relation phases without build planning. +#[pyfunction] +#[pyo3(signature = (**request))] +fn compile_static_command_context<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request: Request<'_, 'py> = Request::new("compile_static_command_context", request)?; + let py: Python<'py> = request.py(); + let none = || py.None().into_bound(py); + let discovered: Bound<'py, PyAny> = request.get("discovered_inputs")?; + let adapter: Bound<'py, PyAny> = request.get("adapter")?; + let options: Bound<'py, PyAny> = + unit_test_selectors_applied(&request.get("options")?, &discovered)?; + let mut selected_keys: Bound<'py, PyAny> = + request.optional("selected_keys")?.unwrap_or_else(none); + if selected_keys.is_none() + && !options + .getattr("sql_test_selection")? + .getattr("models_selected")? + .is_truthy()? + { + selected_keys = PyFrozenSet::empty(py)?.into_any(); + } + let compiled: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "compile_project_phase")?, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ("options", options.clone()), + ( + "on_progress", + request.optional("on_progress")?.unwrap_or_else(none), + ), + ], + )?; + let project: Bound<'py, PyAny> = compiled.getattr("project")?; + let deferred: Bound<'py, PyAny> = + deferred_locations((&discovered, &adapter), &options, &project)?; + let scope: Bound<'py, PyAny> = call_with( + &imported(py, STATIC_SCOPE_MODULE, "resolve_static_command_scope")?, + &[ + ("project", project.clone()), + ( + "selection", + call_with( + &imported(py, PLANNER_MODELS_MODULE, "PlannerSelection")?, + &[ + ("select", options.getattr("select")?), + ("exclude", options.getattr("exclude")?), + ("selected_keys", selected_keys), + ("sql_test_selection", options.getattr("sql_test_selection")?), + ], + )?, + ), + ("auto_load_sources", options.getattr("auto_load_sources")?), + ], + )?; + let relations: Bound<'py, PyAny> = call_with( + &imported( + py, + STATIC_RELATIONS_MODULE, + "resolve_static_relation_context", + )?, + &[ + ("project", project.clone()), + ("adapter", adapter), + ("scope", scope.clone()), + ( + "deferral", + call_with( + &imported(py, PLANNER_MODELS_MODULE, "DeferralInputs")?, + &[ + ("deferred_locations", deferred), + ("defer_sources_to", options.getattr("defer_sources_to")?), + ( + "source_deferral_enabled", + options.getattr("source_deferral_enabled")?, + ), + ], + )?, + ), + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ( + "relation_keys", + request.optional("relation_keys")?.unwrap_or_else(none), + ), + ], + )?; + call_with( + &imported(py, PIPELINE_MODELS_MODULE, "StaticCommandContext")?, + &[ + ("project", project), + ("scope", scope), + ("relations", relations), + ("connection_config", compiled.getattr("connection_config")?), + ("compile_seconds", compiled.getattr("compile_seconds")?), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(apply_unit_test_selectors, module)?)?; + module.add_function(wrap_pyfunction!(resolve_configured_rule_selection, module)?)?; + module.add_function(wrap_pyfunction!(resolve_analysis_model_names, module)?)?; + module.add_function(wrap_pyfunction!(compile_static_command_context, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index 919ddc74f3..ab172b2417 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -18,16 +18,18 @@ use sqlbuild_analysis::assembly::target_checks::main::target_error_message::targ use sqlbuild_analysis::assembly::target_checks::models::{LoaderCollision, MissingParts}; use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_model_config::templates::models::TemplateOptions; + use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; +use crate::bindings::_helpers::model_config::template_data::expanded_template_data; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; -const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; -const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; -const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; -const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; -const EXPAND_TEMPLATE_MODULE: &str = "sqlbuild.compiler.compile.main.expand_template_data"; +const TARGET_NAME_MODULE: &str = "sqlbuild._native"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; +const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild._native"; +const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild._native"; const SETTING_HELP_MODULE: &str = "sqlbuild.errors.setting_help.main.setting_help"; /// Connection settings that identify the warehouse two targets write to. const CONNECTION_IDENTITY_KEYS: [&str; 7] = [ @@ -369,21 +371,17 @@ fn managed_source_namespace( let value: Bound<'_, PyDict> = PyDict::new(py); pyo3::types::PyDictMethods::set_item(&value, "database", database)?; pyo3::types::PyDictMethods::set_item(&value, "schema", schema)?; - let false_value: Bound<'_, PyAny> = pyo3::types::PyBool::new(py, false).to_owned().into_any(); - let expanded: Bound<'_, PyAny> = call_with( - &imported(py, EXPAND_TEMPLATE_MODULE, "expand_template_data")?, - &[ - ("value", value.into_any()), - ("variables", effective_vars.clone()), - ("context_values", PyDict::new(py).into_any()), - ( - "context_label", - PyString::new(py, LOADER_NAMESPACE_LABEL).into_any(), - ), - ("allow_context", false_value.clone()), - ("preserve_context_tokens", false_value.clone()), - ("preserve_unknown_context", false_value), - ], + let expanded: Bound<'_, PyAny> = expanded_template_data( + value.as_any(), + (effective_vars.downcast()?, &PyDict::new(py)), + ( + TemplateOptions { + allow_context: false, + preserve_context_tokens: false, + preserve_unknown_context: false, + }, + LOADER_NAMESPACE_LABEL, + ), )?; let part = |key: &str| -> PyResult> { let value: Bound<'_, PyAny> = expanded.get_item(key)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/expanded_offsets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/expanded_offsets.rs new file mode 100644 index 0000000000..b7a8abb148 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/expanded_offsets.rs @@ -0,0 +1,44 @@ +//! Map offsets in expanded SQL back onto the authored text that produced them. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::compiled_project::main::map_expanded_offset::map_expanded_offset as mapped; +use sqlbuild_analysis::compiled_project::models::SplicedSpan; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; + +/// `map_expanded_offset(*, offset, passes)`: a `MappedOffset` in the first pass's source. +#[pyfunction] +#[pyo3(signature = (*, offset, passes))] +fn map_expanded_offset<'py>( + py: Python<'py>, + offset: usize, + passes: &Bound<'py, PyAny>, +) -> PyResult> { + let mut spans: Vec> = Vec::new(); + for pass in passes.try_iter()? { + let mut pass_spans: Vec = Vec::new(); + for span in pass?.try_iter()? { + let span: Bound<'py, PyAny> = span?; + pass_spans.push(SplicedSpan { + source_start: span.getattr("source_start")?.extract()?, + source_end: span.getattr("source_end")?.extract()?, + output_start: span.getattr("output_start")?.extract()?, + output_end: span.getattr("output_end")?.extract()?, + }); + } + spans.push(pass_spans); + } + let (offset, generated) = mapped(offset, &spans); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("offset", offset)?; + arguments.set_item("generated", generated)?; + py.import(COMPILE_MODELS_MODULE)? + .getattr("MappedOffset")? + .call((), Some(&arguments)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(map_expanded_offset, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index 29c1c178b1..802f7dc128 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod audit_loop; pub(crate) mod audit_views; pub(crate) mod compiled_model_analysis; pub(crate) mod compiled_model_view; +pub(crate) mod expanded_offsets; pub(crate) mod function_loop; pub(crate) mod function_views; pub(crate) mod inputs_view; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs index 93b6bc7006..826d439918 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs @@ -4,5 +4,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::compiled_project::project::register(module)?; crate::bindings::_helpers::compiled_project::model_loop::register(module)?; crate::bindings::_helpers::compiled_project::attachment::register(module)?; + crate::bindings::_helpers::compiled_project::expanded_offsets::register(module)?; crate::bindings::_helpers::compiled_project::records::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index b710376283..d81bf43958 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -18,6 +18,7 @@ use sqlbuild_core::contract_command::models::{ ContractFindingText, ContractProgress, ContractReport, ContractResourceText, }; +use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -27,7 +28,6 @@ const ADAPTER_CONTEXT_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.ada const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; const ADOPTION_MODELS: &str = "sqlbuild.compiler.contract_adoption.models"; const WRITE_MODULE: &str = "sqlbuild.compiler.contract_adoption.main._write"; const GENERATE_ACTION: &str = "generate"; @@ -135,17 +135,7 @@ struct Progress<'py> { impl<'py> Progress<'py> { fn status(&self) -> PyResult> { - let py: Python<'py> = self.stream.py(); - call_with( - &imported(py, STATUS_REPORTER_MODULE, "TransientStatusReporter")?, - &[ - ("stream", self.stream.clone()), - ( - "use_color", - PyBool::new(py, self.use_color).to_owned().into_any(), - ), - ], - ) + status_reporter(&self.stream, self.use_color) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/dynamic_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/dynamic_names.rs deleted file mode 100644 index 3f6575ad8f..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/dynamic_names.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! Compiled dynamic column family `name_pattern`s for matching runtime column names. - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyclass, pymethods}; -use sqlbuild_model_config::name_patterns::main::compile_name_pattern::compile_name_pattern; -use sqlbuild_model_config::name_patterns::models::NamePattern; - -/// A `name_pattern` compiled to match whole column names; an invalid pattern raises `ValueError`. -#[pyclass(name = "NamePattern", module = "sqlbuild._native", frozen)] -struct PyNamePattern { - pattern: NamePattern, -} - -#[pymethods] -impl PyNamePattern { - #[new] - fn new(pattern: &str) -> PyResult { - compile_name_pattern(pattern) - .map(|pattern| Self { pattern }) - .map_err(|error| PyValueError::new_err(format!("invalid name_pattern: {error}"))) - } - - /// Whether `column_name` wholly matches the pattern. - fn matches(&self, column_name: &str) -> bool { - self.pattern.matches(column_name) - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs index 8ec58220c6..18679760d8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs @@ -180,7 +180,7 @@ fn message<'py>( /// `resolve_target_name`: the effective target, `None`, or the unknown-target message. #[pyfunction] -fn effective_config_target_name<'py>( +pub(crate) fn effective_config_target_name<'py>( py: Python<'py>, project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, @@ -202,7 +202,7 @@ fn effective_config_target_name<'py>( /// `resolve_target_config`: each merged `TargetConfig` field, or the ambiguous-connection message. #[pyfunction] -fn effective_config_target_fields<'py>( +pub(crate) fn effective_config_target_fields<'py>( py: Python<'py>, configs: (Bound<'py, PyAny>, Bound<'py, PyAny>), target_name: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs index 38d21f7a13..bd7003c946 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs @@ -1,8 +1,9 @@ pub(crate) mod contract_diagnostics; -pub(crate) mod dynamic_names; pub(crate) mod effective_config; pub(crate) mod model_contracts; pub(crate) mod project_config; pub(crate) mod registration; pub(crate) mod resource_identities; +pub(crate) mod spec_resolution; pub(crate) mod sql_values; +pub(crate) mod state_literals; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs index 846c94987a..cc4f591a2b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs @@ -3,15 +3,16 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::contracts::{ - contract_diagnostics, dynamic_names, effective_config, project_config, resource_identities, - sql_values, + contract_diagnostics, effective_config, project_config, resource_identities, spec_resolution, + sql_values, state_literals, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { contract_diagnostics::register(module)?; - dynamic_names::register(module)?; effective_config::register(module)?; project_config::register(module)?; resource_identities::register(module)?; - sql_values::register(module) + spec_resolution::register(module)?; + sql_values::register(module)?; + state_literals::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs index b890e7365c..7eb4fc6175 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs @@ -1,20 +1,56 @@ -//! Canonical snake_case resource identity checks. +//! Canonical snake_case resource identity checks and SQL function node types. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyString, PyStringMethods}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyString, PyStringMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::resource_identities::main::resource_identity_failure::resource_identity_failure; -/// The snake_case suggestion and help for a non-canonical identity, or `None`. +const RESOURCE_NAME_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.resource_names.exceptions"; +const TABLE_FUNCTION_NODE_TYPE: &str = "table_fn"; +const UDF_NODE_TYPE: &str = "udf"; + +/// Reject one identity unless it uses canonical lowercase snake_case. #[pyfunction] -fn resource_identity_suggestion( +#[pyo3(signature = (*, name, kind, path, private_identity = false))] +fn validate_resource_identity( name: &Bound<'_, PyString>, + kind: &str, + path: &Bound<'_, PyAny>, private_identity: bool, -) -> PyResult> { +) -> Result<(), PyErr> { + let py: Python<'_> = name.py(); let text = name.to_string_lossy(); - Ok(resource_identity_failure(&text, private_identity)) + let Some((corrected_name, help_text)) = resource_identity_failure(&text, private_identity) + else { + return Ok(()); + }; + let message: String = format!( + "Invalid {kind} identity '{text}' in {}; use snake_case '{corrected_name}'", + path.str()? + ); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("help", help_text)?; + let error: Bound<'_, PyAny> = py + .import(RESOURCE_NAME_EXCEPTIONS_MODULE)? + .getattr("ResourceIdentityError")? + .call((message,), Some(&keywords))?; + Err(PyErr::from_value(error)) +} + +/// The public node type of a SQL function: a table function when it returns columns. +#[pyfunction] +#[pyo3(signature = (*, return_columns))] +fn function_node_type(return_columns: &Bound<'_, PyAny>) -> PyResult<&'static str> { + Ok(if return_columns.is_truthy()? { + TABLE_FUNCTION_NODE_TYPE + } else { + UDF_NODE_TYPE + }) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resource_identity_suggestion, module)?) + module.add_function(wrap_pyfunction!(validate_resource_identity, module)?)?; + module.add_function(wrap_pyfunction!(function_node_type, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs new file mode 100644 index 0000000000..097936c985 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs @@ -0,0 +1,343 @@ +//! Effective target, adapter, rendering, scenario and loader destination resolution. + +use std::cell::RefCell; +use std::collections::HashMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_model_config::name_patterns::main::compile_name_pattern::compile_name_pattern; +use sqlbuild_model_config::name_patterns::models::NamePattern; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::contracts::effective_config::{ + effective_config_target_fields, effective_config_target_name, +}; + +const MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const ERRORS_MODULE: &str = "sqlbuild.spec.contracts.exceptions"; +const TARGET_GROUPS: [(&str, &str); 4] = [ + ("clone", "ClonePolicy"), + ("execution_limits", "ExecutionLimitsConfig"), + ("diff", "TargetDiffConfig"), + ("warehouses", "TargetWarehousesConfig"), +]; +const SNAPSHOT_LIMITS: [&str; 4] = [ + "max_rows_per_relation", + "max_total_rows", + "max_bytes_per_relation", + "max_total_bytes", +]; +const ALL_DIALECTS: &str = "*"; +const LOADER_SCHEMA_TABLE_PARTS: usize = 2; +const LOADER_QUALIFIED_TABLE_PARTS: usize = 3; +const NAME_PATTERN_CACHE_LIMIT: usize = 256; + +thread_local! { + static NAME_PATTERNS: RefCell> = RefCell::new(HashMap::new()); +} + +fn spec_error<'py>(py: Python<'py>, message: Bound<'py, PyString>) -> PyResult { + Ok(PyErr::from_value( + imported(py, ERRORS_MODULE, "SpecConfigError")?.call1((message,))?, + )) +} + +/// Resolve the effective target name. +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config, selected_target))] +pub(crate) fn resolve_target_name<'py>( + py: Python<'py>, + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, + selected_target: &Bound<'py, PyAny>, +) -> PyResult>> { + let (name, error) = + effective_config_target_name(py, project_config, local_config, selected_target)?; + match error { + Some(message) => Err(spec_error(py, message)?), + None => Ok(name), + } +} + +/// Merge project target config with local developer overrides. +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config, target_name))] +pub(crate) fn resolve_target_config<'py>( + py: Python<'py>, + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, + target_name: &Bound<'py, PyAny>, +) -> PyResult> { + let target_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "TargetConfig")?; + let (rows, error) = effective_config_target_fields( + py, + (project_config.clone(), local_config.clone()), + target_name, + &target_class.call0()?, + )?; + let Some(rows) = rows else { + let message: Bound<'py, PyString> = error.unwrap_or_else(|| PyString::new(py, "")); + return Err(spec_error(py, message)?); + }; + let values: Bound<'py, PyDict> = PyDict::new(py); + let grouped: Vec<(&str, Bound<'py, PyDict>)> = TARGET_GROUPS + .iter() + .map(|(group, _)| (*group, PyDict::new(py))) + .collect(); + for (group, name, value) in rows { + match group { + None => values.set_item(name, value)?, + Some(group) => { + if let Some((_, fields)) = grouped.iter().find(|(known, _)| *known == group) { + fields.set_item(name, value)?; + } + } + } + } + for ((group, class_name), (_, fields)) in TARGET_GROUPS.iter().zip(&grouped) { + let nested: Bound<'py, PyAny> = + imported(py, MODELS_MODULE, class_name)?.call((), Some(fields))?; + values.set_item(*group, nested)?; + } + target_class.call((), Some(&values)) +} + +/// Resolve the effective adapter name, allowing local override. +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config))] +pub(crate) fn resolve_effective_adapter_name<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, +) -> PyResult> { + let local: Bound<'py, PyAny> = local_config.getattr("adapter")?; + if local.is_none() { + return project_config.getattr("adapter"); + } + Ok(local) +} + +/// Resolve declaration rendering over project configuration. +#[pyfunction] +#[pyo3(signature = (*, project_config, declaration_override))] +pub(crate) fn resolve_effective_collection_rendering<'py>( + project_config: &Bound<'py, PyAny>, + declaration_override: &Bound<'py, PyAny>, +) -> PyResult> { + if !declaration_override.is_none() { + return Ok(declaration_override.clone()); + } + project_config + .getattr("constants")? + .getattr("collection_rendering") +} + +/// `local` when set, else `project`. +fn override_or<'py>(local: Bound<'py, PyAny>, project: Bound<'py, PyAny>) -> Bound<'py, PyAny> { + if local.is_none() { project } else { local } +} + +/// Resolve scenario config, allowing local overrides to replace project rules. +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config))] +fn resolve_effective_scenario_config<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_config.py(); + let project: Bound<'py, PyAny> = project_config.getattr("scenario")?; + let local: Bound<'py, PyAny> = local_config.getattr("scenario")?; + let overrides: Bound<'py, PyDict> = PyDict::new(py); + for item in project + .getattr("local_type_overrides")? + .call_method0("items")? + .try_iter()? + { + let (dialect, rules): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + overrides.set_item(dialect, py.get_type::().call1((rules,))?)?; + } + for item in local + .getattr("local_type_overrides")? + .call_method0("items")? + .try_iter()? + { + let (dialect, rules): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + overrides + .call_method1("setdefault", (dialect, PyDict::new(py)))? + .call_method1("update", (rules,))?; + } + let project_limits: Bound<'py, PyAny> = project.getattr("snapshot_limits")?; + let local_limits: Bound<'py, PyAny> = local.getattr("snapshot_limits")?; + let limits: Vec<(&str, Bound<'py, PyAny>)> = SNAPSHOT_LIMITS + .iter() + .map(|field| { + Ok(( + *field, + override_or( + local_limits.getattr(*field)?, + project_limits.getattr(*field)?, + ), + )) + }) + .collect::>()?; + call_with( + &imported(py, MODELS_MODULE, "ScenarioConfig")?, + &[ + ( + "run_namespace", + override_or( + local.getattr("run_namespace")?, + project.getattr("run_namespace")?, + ), + ), + ("local_type_overrides", overrides.into_any()), + ( + "snapshot_limits", + call_with( + &imported(py, MODELS_MODULE, "ScenarioSnapshotLimitsConfig")?, + &limits, + )?, + ), + ], + ) +} + +/// Return global and dialect-specific scenario local type override rules. +#[pyfunction] +#[pyo3(signature = (*, scenario_config, sql_analysis_dialect))] +fn scenario_local_type_overrides_for_dialect<'py>( + scenario_config: &Bound<'py, PyAny>, + sql_analysis_dialect: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = scenario_config.py(); + let rules: Bound<'py, PyAny> = scenario_config.getattr("local_type_overrides")?; + let overrides: Bound<'py, PyDict> = PyDict::new(py); + overrides.call_method1( + "update", + (rules.call_method1("get", (ALL_DIALECTS, PyDict::new(py)))?,), + )?; + if !sql_analysis_dialect.is_none() { + overrides.call_method1( + "update", + (rules.call_method1("get", (sql_analysis_dialect, PyDict::new(py)))?,), + )?; + } + Ok(overrides) +} + +/// Whether `text` is empty or only whitespace as Python's `str.strip` sees it. +fn blank(text: &str) -> bool { + text.chars() + .all(|character| character.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&character)) +} + +/// Parse a one-, two-, or three-part loader destination. +#[pyfunction] +#[pyo3(signature = (*, destination, default_database, default_schema))] +fn loader_destination_parts<'py>( + py: Python<'py>, + destination: &str, + default_database: Option>, + default_schema: Option>, +) -> PyResult> { + loader_parts(py, destination, (default_database, default_schema)) +} + +fn loader_parts<'py>( + py: Python<'py>, + destination: &str, + (default_database, default_schema): (Option>, Option>), +) -> PyResult> { + let parts: Vec<&str> = destination.split('.').collect(); + if !(1..=LOADER_QUALIFIED_TABLE_PARTS).contains(&parts.len()) + || parts.iter().any(|part| blank(part)) + { + return Err(spec_error( + py, + PyString::new( + py, + &format!( + "Invalid loader destination '{destination}': expected 1 to 3 non-empty \ + dot-separated parts (table, schema.table, or database.schema.table)" + ), + ), + )?); + } + let none = || py.None().into_bound(py); + let text = |part: &str| PyString::new(py, part).into_any(); + let (database, schema, table): (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>) = + match parts.as_slice() { + [table] => ( + default_database.unwrap_or_else(none), + default_schema.unwrap_or_else(none), + text(table), + ), + [schema, table] if parts.len() == LOADER_SCHEMA_TABLE_PARTS => ( + default_database.unwrap_or_else(none), + text(schema), + text(table), + ), + _ => (text(parts[0]), text(parts[1]), text(parts[2])), + }; + imported(py, MODELS_MODULE, "LoaderDestinationParts")?.call1((database, schema, table)) +} + +/// Whether a family's optional `name_pattern` accepts `column_name`. +fn family_accepts(family: &Bound<'_, PyAny>, column_name: &str) -> PyResult { + let pattern: Option = family.getattr("name_pattern")?.extract()?; + let Some(pattern) = pattern else { + return Ok(true); + }; + NAME_PATTERNS.with(|cache| { + let mut cache = cache.borrow_mut(); + if !cache.contains_key(&pattern) { + let compiled: NamePattern = compile_name_pattern(&pattern) + .map_err(|error| PyValueError::new_err(format!("invalid name_pattern: {error}")))?; + if cache.len() >= NAME_PATTERN_CACHE_LIMIT { + cache.clear(); + } + cache.insert(pattern.clone(), compiled); + } + Ok(cache + .get(&pattern) + .is_some_and(|compiled| compiled.matches(column_name))) + }) +} + +/// Return declared dynamic families accepting a physical column name. +#[pyfunction] +#[pyo3(signature = (*, families, column_name))] +fn matching_dynamic_families<'py>( + families: &Bound<'py, PyAny>, + column_name: &str, +) -> PyResult> { + let mut matching: Vec> = Vec::new(); + for family in families.try_iter()? { + let family: Bound<'py, PyAny> = family?; + if family_accepts(&family, column_name)? { + matching.push(family); + } + } + PyTuple::new(families.py(), matching) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_target_name, module)?)?; + module.add_function(wrap_pyfunction!(resolve_target_config, module)?)?; + module.add_function(wrap_pyfunction!(resolve_effective_adapter_name, module)?)?; + module.add_function(wrap_pyfunction!( + resolve_effective_collection_rendering, + module + )?)?; + module.add_function(wrap_pyfunction!(resolve_effective_scenario_config, module)?)?; + module.add_function(wrap_pyfunction!( + scenario_local_type_overrides_for_dialect, + module + )?)?; + module.add_function(wrap_pyfunction!(loader_destination_parts, module)?)?; + module.add_function(wrap_pyfunction!(matching_dynamic_families, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/sql_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/sql_values.rs index c289f13901..aea1abc5b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/sql_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/sql_values.rs @@ -10,7 +10,7 @@ use pyo3::types::{ use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_core::models::{SqlLogicalType, SqlValue}; use sqlbuild_model_config::sql_values::errors::SqlValueFailure; -use sqlbuild_model_config::sql_values::main::normalize_sql_value::normalize_sql_value; +use sqlbuild_model_config::sql_values::main::normalize_sql_value::normalize_sql_value as normalized; use sqlbuild_model_config::sql_values::models::{RawKey, RawSqlValue, SqlValueLimits}; const MODELS_MODULE: &str = "sqlbuild.sql_values.models"; @@ -191,27 +191,34 @@ pub(crate) fn sql_value_error(py: Python<'_>, failure: SqlValueFailure) -> PyRes /// `normalize_sql_value`: the canonical `SqlValue` of an authored value. #[pyfunction] -fn normalize_typed_sql_value<'py>( - py: Python<'py>, +#[pyo3(signature = (*, raw_value, context, explicit_type = None, limits = None))] +fn normalize_sql_value<'py>( raw_value: &Bound<'py, PyAny>, - typing: (String, Option), - limits: (i64, i64, i64), + context: &str, + explicit_type: Option<&str>, + limits: Option<&Bound<'py, PyAny>>, ) -> PyResult> { - let (context, explicit_type) = typing; + let py: Python<'py> = raw_value.py(); let classes = SqlValueClasses::load(py)?; let raw = self::raw_value(raw_value, &classes)?; - let (max_depth, max_elements, max_size) = limits; + let applied: Bound<'py, PyAny> = match limits { + Some(limits) if !limits.is_none() => limits.clone(), + _ => py + .import(MODELS_MODULE)? + .getattr("SqlValueLimits")? + .call0()?, + }; let limits = SqlValueLimits { - max_depth, - max_elements, - max_size, + max_depth: applied.getattr("max_depth")?.extract()?, + max_elements: applied.getattr("max_elements")?.extract()?, + max_size: applied.getattr("max_size")?.extract()?, }; - match normalize_sql_value(&raw, &context, explicit_type.as_deref(), limits) { + match normalized(&raw, context, explicit_type, limits) { Ok(value) => sql_value(&value, &classes), Err(failure) => Err(sql_value_error(py, failure)?), } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(normalize_typed_sql_value, module)?) + module.add_function(wrap_pyfunction!(normalize_sql_value, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs new file mode 100644 index 0000000000..58635c12f9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs @@ -0,0 +1,158 @@ +//! Portable typed SQL literals for warehouse-state columns. + +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyInt, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; + +const EXCEPTIONS_MODULE: &str = "sqlbuild.sql_values.exceptions"; +const DATETIME_MODULE: &str = "datetime"; +const JSON_MODULE: &str = "json"; +const STRING_TYPE: &str = "string"; +const INTEGER_TYPE: &str = "integer"; +const BOOLEAN_TYPE: &str = "boolean"; +const TIMESTAMP_TYPE: &str = "timestamp"; +const TEXT_TIMESTAMP_TYPE: &str = "text_timestamp"; +const DATE_TYPE: &str = "date"; +const JSON_TYPE: &str = "json"; + +/// `render_state_sql_literal(*, value, declared_type)`: `value` as its state column's literal. +#[pyfunction] +#[pyo3(signature = (*, value, declared_type))] +fn render_state_sql_literal( + py: Python<'_>, + value: &Bound<'_, PyAny>, + declared_type: &str, +) -> PyResult { + if value.is_none() { + return Ok(typed_null(declared_type).to_owned()); + } + match declared_type { + STRING_TYPE => { + if !value.is_instance_of::() { + return Err(rendering_error( + py, + "state string columns require str values", + )); + } + Ok(quoted(&value.extract::()?)) + } + INTEGER_TYPE => { + if value.is_instance_of::() || !value.is_instance_of::() { + return Err(rendering_error( + py, + "state integer columns require int values", + )); + } + let rendered: String = value.call_method1("__format__", ("",))?.extract()?; + Ok(format!("CAST({rendered} AS BIGINT)")) + } + BOOLEAN_TYPE => { + if !value.is_instance_of::() { + return Err(rendering_error( + py, + "state boolean columns require bool values", + )); + } + Ok(if value.is_truthy()? { "TRUE" } else { "FALSE" }.to_owned()) + } + TIMESTAMP_TYPE | TEXT_TIMESTAMP_TYPE => timestamp_literal(value, declared_type), + DATE_TYPE => { + let datetime = py.import(DATETIME_MODULE)?; + if value.is_instance(&datetime.getattr("datetime")?)? + || !value.is_instance(&datetime.getattr("date")?)? + { + return Err(rendering_error( + py, + "state date columns require date values", + )); + } + let iso: String = value.call_method0("isoformat")?.extract()?; + Ok(format!("CAST({} AS DATE)", quoted(&iso))) + } + JSON_TYPE => json_literal(value), + other => Err(PyValueError::new_err(format!( + "unknown state SQL value type {other:?}" + ))), + } +} + +fn timestamp_literal(value: &Bound<'_, PyAny>, declared_type: &str) -> PyResult { + let py: Python<'_> = value.py(); + let datetime = py.import(DATETIME_MODULE)?; + if !value.is_instance(&datetime.getattr("datetime")?)? { + return Err(rendering_error( + py, + "state timestamp columns require datetime values", + )); + } + let utc: Bound<'_, PyAny> = datetime.getattr("UTC")?; + let normalized: Bound<'_, PyAny> = + if value.getattr("tzinfo")?.is_none() || value.call_method0("utcoffset")?.is_none() { + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("tzinfo", &utc)?; + value.call_method("replace", (), Some(&arguments))? + } else { + value.call_method1("astimezone", (&utc,))? + }; + let rendered: String = quoted(&normalized.call_method0("isoformat")?.extract::()?); + if declared_type == TEXT_TIMESTAMP_TYPE { + return Ok(rendered); + } + Ok(format!("CAST({rendered} AS TIMESTAMP)")) +} + +fn json_literal(value: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = value.py(); + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("ensure_ascii", false)?; + arguments.set_item("separators", (",", ":"))?; + arguments.set_item("sort_keys", true)?; + match py + .import(JSON_MODULE)? + .getattr("dumps")? + .call((value,), Some(&arguments)) + { + Ok(serialized) => Ok(quoted(&serialized.extract::()?)), + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + let raised: PyErr = + rendering_error(py, "state JSON columns require JSON-serializable values"); + raised.set_cause(py, Some(error)); + Err(raised) + } + Err(error) => Err(error), + } +} + +fn quoted(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +fn typed_null(declared_type: &str) -> &'static str { + match declared_type { + INTEGER_TYPE => "CAST(NULL AS BIGINT)", + TIMESTAMP_TYPE => "CAST(NULL AS TIMESTAMP)", + DATE_TYPE => "CAST(NULL AS DATE)", + _ => "NULL", + } +} + +fn rendering_error(py: Python<'_>, message: &str) -> PyErr { + let raised = || -> PyResult { + let error: Bound<'_, PyAny> = py + .import(EXCEPTIONS_MODULE)? + .getattr("SqlValueRenderingError")? + .call1((message,))?; + Ok(PyErr::from_value(error)) + }; + raised().unwrap_or_else(|error| error) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_state_sql_literal, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs index f979330cd9..073bca79ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs @@ -12,8 +12,8 @@ use sqlbuild_core::cost_command::models::CostCoverage; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild.compiler.compile.main._effective_target"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild._native"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const COST_TYPES: &str = "sqlbuild.cost.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs index a31d7399ca..7bd4590a5c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs @@ -52,12 +52,10 @@ fn duration_span(duration: &Bound<'_, PyAny>) -> PyResult { }) } -/// `moment` moved forward (`direction` 1) or back (`direction` -1) by `duration`. -#[pyfunction] -#[pyo3(signature = (*, moment, duration, direction))] -fn shift_by_cursor_duration<'py>( +/// `moment` (a `date` or `datetime`) moved by `span` in `direction`, keeping its type and `tzinfo`. +pub(crate) fn shifted_moment<'py>( moment: &Bound<'py, PyAny>, - duration: &Bound<'py, PyAny>, + span: &DurationSpan, direction: i64, ) -> PyResult> { let py: Python<'py> = moment.py(); @@ -71,11 +69,7 @@ fn shift_by_cursor_duration<'py>( } else { Moment::Date(civil_date(moment)?) }; - let shifted: Moment = session.checked(shift_by_duration( - typed, - &duration_span(duration)?, - direction, - ))?; + let shifted: Moment = session.checked(shift_by_duration(typed, span, direction))?; match shifted { Moment::Timestamp { local, fold } => replaced(moment, &local, fold), Moment::Date(date) => call_with( @@ -89,6 +83,17 @@ fn shift_by_cursor_duration<'py>( } } +/// `moment` moved forward (`direction` 1) or back (`direction` -1) by `duration`. +#[pyfunction] +#[pyo3(signature = (*, moment, duration, direction))] +fn shift_by_cursor_duration<'py>( + moment: &Bound<'py, PyAny>, + duration: &Bound<'py, PyAny>, + direction: i64, +) -> PyResult> { + shifted_moment(moment, &duration_span(duration)?, direction) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(parse_cursor_duration, module)?)?; module.add_function(wrap_pyfunction!(shift_by_cursor_duration, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs index 01348c309c..ea7f60a7c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs @@ -3,10 +3,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyBool; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::compile_driver::python_calls::imported; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; /// `sys.stdout` or `sys.stderr`, looked up when written. pub(crate) fn stream<'py>(py: Python<'py>, name: &str) -> PyResult> { @@ -45,12 +45,5 @@ pub(crate) fn status<'py>( stream: &Bound<'py, PyAny>, use_color: bool, ) -> PyResult> { - let py: Python<'py> = stream.py(); - call_with( - &imported(py, STATUS_REPORTER_MODULE, "TransientStatusReporter")?, - &[ - ("stream", stream.clone()), - ("use_color", flag(py, use_color)), - ], - ) + status_reporter(stream, use_color) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 84ad9cafc5..069ca675fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -35,7 +35,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const DEBUG_OPTIONS: (&[&str], &[&str]) = ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs index bd17fa0151..7b92d284e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -31,8 +31,8 @@ use crate::bindings::_helpers::frontier::encoder::emit_error; const NATIVE_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; -const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const TARGET_NAME_MODULE: &str = "sqlbuild._native"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const CONTRACT_CONSTANTS_MODULE: &str = "sqlbuild.spec.contracts.constants"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const EXECUTE_MODULE: &str = "sqlbuild.executor.diff.main.execute"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index 6efc6275fc..ad318afc21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -18,10 +18,10 @@ use crate::bindings::_helpers::diff_report::support::{ }; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; -const NAMESPACE_MODULE: &str = "sqlbuild.compiler.compile.main._effective_target_namespace"; -const RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; +const NAMESPACE_MODULE: &str = "sqlbuild._native"; +const RUNTIME_MODULE: &str = "sqlbuild._native"; const POLYGLOT_MODULE: &str = "sqlbuild.compiler.sql_analysis.main.import_polyglot_sql"; const COMMAND_MODELS_MODULE: &str = "sqlbuild.cli.commands.models"; const LIFECYCLE_MODULE: &str = "sqlbuild.executor.diff.classes.query_artifact_lifecycle"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs index c0707d7764..c32f91ad2d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs @@ -15,6 +15,7 @@ use sqlbuild_core::format_command::models::LintReport; use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_null_candidates; use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -26,7 +27,6 @@ use crate::bindings::_helpers::format_command::selection::{LintInputs, resolve_l use crate::bindings::_helpers::frontier::encoder::emit_error; const NATIVE_MODULE: &str = "sqlbuild._native"; -const STATUS_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; const DEFAULT_DIALECT: &str = "generic"; @@ -166,11 +166,8 @@ fn render_diff(py: Python<'_>, changes: &[FormatChange<'_>]) -> PyResult<()> { Ok(()) } -fn status_reporter<'py>(py: Python<'py>, use_color: bool) -> PyResult> { - let options: Bound<'py, PyDict> = PyDict::new(py); - options.set_item("stream", py.import("sys")?.getattr("stderr")?)?; - options.set_item("use_color", use_color)?; - imported(py, STATUS_MODULE, "TransientStatusReporter")?.call((), Some(&options)) +fn stderr_status<'py>(py: Python<'py>, use_color: bool) -> PyResult> { + status_reporter(&py.import("sys")?.getattr("stderr")?, use_color) } fn format_with_status<'py>( @@ -179,7 +176,7 @@ fn format_with_status<'py>( use_color: bool, ) -> PyResult<(FormatOutcome<'py>, Bound<'py, PyAny>, f64)> { let started: Instant = Instant::now(); - let status: Bound<'py, PyAny> = status_reporter(py, use_color)?; + let status: Bound<'py, PyAny> = stderr_status(py, use_color)?; status.call_method1("start", (START_MESSAGE,))?; match run_format(request) { Ok(outcome) => Ok((outcome, status, started.elapsed().as_secs_f64())), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs index 85bc4e2fc9..7f607c3774 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs @@ -6,8 +6,9 @@ use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; use pyo3::exceptions::PyOSError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; const LINE_FEED: &str = "\n"; const CARRIAGE_RETURN_LINE_FEED: &str = "\r\n"; @@ -87,6 +88,17 @@ fn write_atomically(path: &Path, contents: &str) -> PyResult<()> { written.map_err(|error| PyOSError::new_err(format!("{}: {error}", path.display()))) } +/// `write_atomically(*, path, contents)`: replace a text file atomically, keeping its mode. +#[pyfunction] +#[pyo3(name = "write_atomically", signature = (*, path, contents))] +fn write_file_atomically(path: PathBuf, contents: &str) -> PyResult<()> { + write_atomically(&path, contents) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(write_file_atomically, module)?) +} + /// The changed files in path order, written in place when `write` is set. pub(crate) fn persist_changes<'py>( (files, updated): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/registration.rs index c2bf6cb02c..1b00100a44 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/registration.rs @@ -1,9 +1,10 @@ -//! Register the native `sqb format` command. +//! Register the native `sqb format` command and its atomic file writer. use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::format_command::command; +use crate::bindings::_helpers::format_command::{command, files}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - command::register(module) + command::register(module)?; + files::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index 1508e8bbdf..01508c0bdf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -23,7 +23,7 @@ const UNIT_TEST_SELECTORS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const SELECTION_CODE: &str = "S007"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index 2bc4885d43..a6f4a96e63 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -22,7 +22,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const CONNECTION_MODULE: &str = "sqlbuild._native"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs index dec05b79b1..8b838282a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs @@ -14,6 +14,8 @@ use crate::bindings::_helpers::frontier::encoder::{CaptureEncoder, emit_error}; const CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR"; const CAPTURE_SUFFIX: &str = ".json"; +const PLAN_CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_PLANNER_CAPTURE_DIR"; +const PLAN_CAPTURE_NAME: &str = "plan_output"; const FRONTIER_CONSTANTS_MODULE: &str = "sqlbuild.compiler.frontier.constants"; /// The capture document of `value`, or the shared nodes written before encoding failed. @@ -55,6 +57,36 @@ pub(crate) fn write_stage_capture( "__truediv__", (format!("{number:03}-{stage_name}{CAPTURE_SUFFIX}"),), )?; + write_capture_file(&path, value) +} + +/// Write the planner's `PlanOutput` when SQLBUILD_PLANNER_CAPTURE_DIR names a directory. +pub(crate) fn write_plan_capture(plan_output: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = plan_output.py(); + let directory: Bound<'_, PyAny> = + imported(py, "os", "environ")?.call_method1("get", (PLAN_CAPTURE_DIR_ENV_VAR,))?; + if !directory.is_truthy()? { + return Ok(()); + } + let number: u64 = imported(py, "builtins", "next")? + .call1((imported( + py, + FRONTIER_CONSTANTS_MODULE, + "STAGE_CAPTURE_SEQUENCE", + )?,))? + .extract()?; + let path: Bound<'_, PyAny> = imported(py, "pathlib", "Path")? + .call1((directory,))? + .call_method1( + "__truediv__", + (format!("{number:03}-{PLAN_CAPTURE_NAME}{CAPTURE_SUFFIX}"),), + )?; + write_capture_file(&path, plan_output) +} + +/// Write the capture document of `value` to `path`, creating its directory. +fn write_capture_file(path: &Bound<'_, PyAny>, value: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = path.py(); call_with( &path.getattr("parent")?.getattr("mkdir")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs index ce588a9c1e..54c8e6f8f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs @@ -11,7 +11,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; const BUILTINS_MODULE: &str = "builtins"; /// Keys within `max_depth` edges of any start, plus the starts when `include_starts`. -fn closure<'py>( +pub(crate) fn closure<'py>( starts: Vec>, edges: &Bound<'py, PyAny>, (include_starts, max_depth): (bool, Option), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs index 5e3f7e22ae..e33df4195c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs @@ -1,7 +1,8 @@ -mod algorithms; +pub(crate) mod algorithms; mod execution_gates; pub(crate) mod keys; pub(crate) mod project_graph; +mod python_lifecycle; pub(crate) mod python_selection; pub(crate) mod registration; pub(crate) mod views; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs new file mode 100644 index 0000000000..f6b6143dff --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs @@ -0,0 +1,80 @@ +//! Lifecycle regions of a run's selected Python nodes, classified natively. + +use std::collections::{BTreeSet, HashMap}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::python_plan::main::run_lifecycle::run_lifecycle; +use sqlbuild_analysis::python_plan::models::{LifecycleNode, RunLifecycle}; + +const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.compiler.python_nodes.models"; +const EXTERNAL_CONNECTION_MODE: &str = "external"; + +/// `build_python_sql_run_lifecycle(*, selection, python_graph)`: a `PythonSqlRunLifecyclePlan`. +#[pyfunction] +#[pyo3(signature = (*, selection, python_graph))] +fn build_python_sql_run_lifecycle<'py>( + selection: &Bound<'py, PyAny>, + python_graph: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = selection.py(); + let selected: BTreeSet = selection + .getattr("python_node_names")? + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?; + let mut nodes: HashMap = HashMap::new(); + for item in python_graph + .getattr("nodes_by_name")? + .call_method0("items")? + .try_iter()? + { + let (name, node): (String, Bound<'py, PyAny>) = item?.extract()?; + nodes.insert(name, lifecycle_node(&node)?); + } + let mut upstream: HashMap> = HashMap::new(); + for item in python_graph + .getattr("upstream_deps")? + .call_method0("items")? + .try_iter()? + { + let (name, deps): (String, Bound<'py, PyAny>) = item?.extract()?; + let deps: Vec = deps + .try_iter()? + .map(|dep| dep?.extract()) + .collect::>()?; + upstream.insert(name, deps); + } + let lifecycle: RunLifecycle = py.detach(|| run_lifecycle(&selected, &nodes, &upstream)); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item( + "ingress_python_node_names", + PyFrozenSet::new(py, lifecycle.ingress_names)?, + )?; + arguments.set_item( + "ingress_loader_names", + PyFrozenSet::new(py, lifecycle.ingress_loader_names)?, + )?; + arguments.set_item( + "read_side_python_node_names", + PyFrozenSet::new(py, lifecycle.read_side_names)?, + )?; + py.import(PYTHON_NODE_MODELS_MODULE)? + .getattr("PythonSqlRunLifecyclePlan")? + .call((), Some(&arguments)) +} + +fn lifecycle_node(node: &Bound<'_, PyAny>) -> PyResult { + let loader: Bound<'_, PyAny> = node.getattr("loader")?; + let external_loader: bool = !loader.is_none() + && loader.getattr("connection_mode")?.extract::()? == EXTERNAL_CONNECTION_MODE; + Ok(LifecycleNode { + kind: node.getattr("kind")?.extract()?, + external_loader, + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_python_sql_run_lifecycle, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs index 485560f171..f9e662f18c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs @@ -5,5 +5,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::graph::algorithms::register(module)?; crate::bindings::_helpers::graph::execution_gates::register(module)?; crate::bindings::_helpers::graph::python_selection::register(module)?; + crate::bindings::_helpers::graph::python_lifecycle::register(module)?; crate::bindings::_helpers::graph::views::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs index 0ca023d5c2..3f3ce3f1e5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs @@ -164,7 +164,7 @@ fn native_graph_from_views(request: Option<&Bound<'_, PyDict>>) -> PyResult, downstream: &Bound<'_, PyAny>, ) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index eb83130059..6d3bc6bb69 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -26,7 +26,7 @@ const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const PREVIEW_TARGET_MODULE: &str = "sqlbuild._native"; -const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; +const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const RETENTION_MODULE: &str = "sqlbuild.executor.janitor.main.resolve_retention_days"; const PLANNING_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.planning"; const EXECUTION_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.execution"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index a1fdf1b70f..0296e7af8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -41,7 +41,7 @@ const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const STATUS_MODULE: &str = "sqlbuild.presentation.main._maybe_status"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; const SELECTION_MODULE: &str = "sqlbuild.compiler.planner.main.selection.selection"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 931f832f8e..54353b4c43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -28,10 +28,10 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild.compiler.compile.main._effective_target"; -const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; -const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild.compiler.compile.main.effective_settings"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild._native"; +const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild._native"; +const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild._native"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; const CURSOR_OVERRIDES_MODULE: &str = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 62c40b9cbe..7b66a5d255 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -18,9 +18,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; -const DESTINATION_MODULE: &str = "sqlbuild.spec.contracts.main.loader_destination_parts"; -const CONNECTION_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const DESTINATION_MODULE: &str = "sqlbuild._native"; +const CONNECTION_CONFIG_MODULE: &str = "sqlbuild._native"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTER_TYPES_MODULE: &str = "sqlbuild.adapter.contract.types"; const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; const LOADER_TABLE_PREFIX: &str = "__loader__"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs index a4bfb862b6..d2bfcafadb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs @@ -4,21 +4,25 @@ use pyo3::exceptions::{PyKeyError, PyRecursionError, PyTypeError, PyValueError}; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::PyBytes; -use pyo3::{FromPyObject, PyRef, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::types::{PyBytes, PyDict}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_analysis::manifest::errors::{ManifestEncodeError, ManifestError}; use sqlbuild_analysis::manifest::main::build_manifest_json::build_manifest_json; use sqlbuild_analysis::manifest::models::ManifestInput; use sqlbuild_core::json::errors::JsonEmitError; +use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::bindings::_helpers::graph::views::native_graph_from_edges; use crate::bindings::_helpers::manifest::rows::{ ProjectRows, plan_input, project_rows, python_hooks, }; const FINGERPRINT_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.fingerprints.exceptions"; +const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; +const DATETIME_MODULE: &str = "datetime"; /// A built manifest: its `manifest.json` bytes, or the error `json.dumps` raised encoding it. #[pyclass(module = "sqlbuild._native", frozen)] @@ -56,36 +60,41 @@ fn python_error(error: ManifestError, py: Python<'_>) -> PyErr { } } -/// The manifest facts only Python knows: names, the generation stamp and hook identities. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct ManifestFacts<'py> { - project_name: String, - adapter_type: String, - /// `datetime.now(tz=UTC).isoformat()`. - generated_at: String, - /// `build_hook_identities(project.hook_functions)`. - hook_identities: Bound<'py, PyAny>, -} - -/// The manifest of `project` and the optional plan. +/// The manifest of `project` and the optional plan, over `graph` or `upstream_deps`/`downstream_deps`. #[pyfunction] -#[pyo3(signature = (*, project, plan_output, graph, facts))] -pub(crate) fn build_manifest( - project: &Bound<'_, PyAny>, - plan_output: &Bound<'_, PyAny>, - graph: PyRef<'_, NativeProjectGraph>, - facts: ManifestFacts<'_>, -) -> PyResult> { +#[pyo3(signature = (**options))] +pub(crate) fn build_manifest(options: Option<&Bound<'_, PyDict>>) -> PyResult> { + let request = Request::new("build_manifest", options)?; + let py: Python<'_> = request.py(); + let project: Bound<'_, PyAny> = request.get("project")?; + let plan_output: Bound<'_, PyAny> = request + .optional("plan_output")? + .unwrap_or_else(|| py.None().into_bound(py)); + let project_name: String = request.get("project_name")?.extract()?; + let adapter_type: String = request.get("adapter_type")?.extract()?; + let graph: Bound<'_, NativeProjectGraph> = match request.optional("graph")? { + Some(graph) if !graph.is_none() => graph.cast_into()?, + _ => Bound::new( + py, + native_graph_from_edges( + &request.get("upstream_deps")?, + &request.get("downstream_deps")?, + )?, + )?, + }; + let utc: Bound<'_, PyAny> = py.import(DATETIME_MODULE)?.getattr("UTC")?; + let generated_at: String = py + .import(DATETIME_MODULE)? + .getattr("datetime")? + .call_method1("now", (utc,))? + .call_method0("isoformat")? + .extract()?; + let hook_identities: Bound<'_, PyAny> = py + .import(HOOK_IDENTITIES_MODULE)? + .getattr("build_hook_identities")? + .call1((project.getattr("hook_functions")?,))?; compiler_guard(|| { - let py: Python<'_> = project.py(); - let ManifestFacts { - project_name, - adapter_type, - generated_at, - hook_identities, - } = facts; - let python_hooks = python_hooks(project, &hook_identities)?; + let python_hooks = python_hooks(&project, &hook_identities)?; let ProjectRows { run_id, sql_analysis_dialect, @@ -95,7 +104,7 @@ pub(crate) fn build_manifest( macros, macro_declarations, sql_hooks, - } = project_rows(project)?; + } = project_rows(&project)?; let input: ManifestInput = ManifestInput { project_name, adapter_type, @@ -109,8 +118,9 @@ pub(crate) fn build_manifest( macro_declarations, sql_hooks, python_hooks, - plan: plan_input(plan_output)?, + plan: plan_input(&plan_output)?, }; + let graph = graph.borrow(); let graph: &ProjectGraph = graph.graph(); let contents = py .detach(|| build_manifest_json(&input, graph)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 8b309cf090..71b7bb0094 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod auditing; pub(crate) mod authored_values; pub(crate) mod boundary; pub(crate) mod build_command; +pub(crate) mod build_progress; pub(crate) mod cache; pub(crate) mod check; pub(crate) mod cli; @@ -43,6 +44,8 @@ pub(crate) mod macro_bridge; pub(crate) mod manifest; pub(crate) mod model_config; pub(crate) mod plan_output; +pub(crate) mod planner_changes; +pub(crate) mod planner_strategy; pub(crate) mod presentation_text; pub(crate) mod project_assembly; pub(crate) mod python_values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs index 4efb336efe..e0375e37ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs @@ -3,12 +3,12 @@ use std::cell::RefCell; use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyResult, Python}; use pyo3::types::{ PyBool, PyBoolMethods, PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, PyTupleMethods, }; -use pyo3::{FromPyObject, IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; +use pyo3::{FromPyObject, PyErr}; use sqlbuild_model_config::errors::ConfigError; use sqlbuild_model_config::templates::errors::TemplateError; use sqlbuild_model_config::templates::main::expand_template_string::expand_template_string; @@ -20,6 +20,7 @@ use sqlbuild_model_config::templates::models::{ use sqlbuild_model_config::templates::types::TemplateHost; use crate::bindings::_helpers::model_config::config_errors::native_config_error; +use crate::bindings::_helpers::model_config::errors::NativeConfigError; const TEMPLATE_OPEN_TOKEN: &str = "${"; const ENVIRONMENT_READ: &str = "env"; @@ -155,10 +156,6 @@ pub(crate) struct TemplateSources<'py>( pub(crate) Bound<'py, PyDict>, ); -/// `allow_context`, `preserve_context_tokens`, `preserve_unknown_context` and the context label. -#[derive(FromPyObject)] -struct TemplateFlags(bool, bool, bool, String); - /// A template error, or a Python error raised while reading values or building containers. pub(crate) enum Stop { Failure(TemplateError), @@ -268,31 +265,28 @@ impl<'py> TemplateHost for PythonHost<'py> { } } -/// Return `(value, reads)` as `expand_template_data` would, or `(error, reads)`. -#[pyfunction] -fn expand_config_templates<'py>( +/// The expanded value, or the native error expansion stopped at, and the reads in order. +pub(crate) type TemplateOutcome<'py> = ( + Result, Py>, + Vec<(&'static str, String)>, +); + +/// `value` with every template expanded under `options`, or the error, with its reads. +pub(crate) fn expand_config_templates<'py>( py: Python<'py>, - value: Bound<'py, PyAny>, + value: &Bound<'py, PyAny>, sources: TemplateSources<'py>, - flags: TemplateFlags, -) -> PyResult> { - let TemplateFlags(allow_context, preserve_context_tokens, preserve_unknown_context, label) = - flags; - let options = TemplateOptions { - allow_context, - preserve_context_tokens, - preserve_unknown_context, - }; - let host = PythonHost::new(py, sources, &label); - let result = expanded(&host, &value, options); - outcome(py, host, result, &label) + (options, label): (TemplateOptions, &str), +) -> PyResult> { + let host = PythonHost::new(py, sources, label); + let result = expanded(&host, value, options); + outcome(py, host, result, label) } /// The first `(var name, value path, UTF-8 byte offset)` of a lone surrogate in raw vars. -#[pyfunction] -fn lone_surrogate_site( - raw_values: Bound<'_, PyDict>, -) -> PyResult, String, usize)>> { +pub(crate) fn lone_surrogate_site<'py>( + raw_values: &Bound<'py, PyDict>, +) -> PyResult, String, usize)>> { for (key, value) in raw_values.iter() { if let Some((path, offset)) = surrogate_path(&value, VALUE_PATH.to_owned())? { return Ok(Some((key.str()?, path, offset))); @@ -348,13 +342,12 @@ fn first_surrogate_offset(text: &Bound<'_, PyString>) -> PyResult> Ok(None) } -/// Return `(resolved vars, reads)` as `expand_effective_vars` would, or `(error, reads)`. -#[pyfunction] -fn expand_effective_vars<'py>( +/// The resolved effective vars, or the native error resolution stopped at, with the reads. +pub(crate) fn resolve_effective_vars<'py>( py: Python<'py>, - raw_values: Bound<'py, PyDict>, + raw_values: &Bound<'py, PyDict>, environment: Bound<'py, PyAny>, -) -> PyResult> { +) -> PyResult> { let mut host = PythonHost::new( py, TemplateSources(raw_values.clone(), environment, PyDict::new(py)), @@ -381,18 +374,13 @@ fn outcome<'py>( host: PythonHost<'py>, result: Result, Stop>, label: &str, -) -> PyResult> { +) -> PyResult> { let value = match result { - Ok(value) => value, - Err(Stop::Failure(error)) => native_config_error(py, template_error(&error, label))? - .into_pyobject(py)? - .into_any(), + Ok(value) => Ok(value), + Err(Stop::Failure(error)) => Err(native_config_error(py, template_error(&error, label))?), Err(Stop::Python(error)) => return Err(error), }; - Ok((value, host.into_reads()) - .into_pyobject(py)? - .into_any() - .unbind()) + Ok((value, host.into_reads())) } /// Expand every template in `value` and the containers inside it. @@ -443,10 +431,3 @@ pub(crate) fn expanded<'py>( } Ok(value.clone()) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(expand_config_templates, module)?)?; - module.add_function(wrap_pyfunction!(expand_effective_vars, module)?)?; - module.add_function(wrap_pyfunction!(lone_surrogate_site, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs index ddae3fc318..26f364aa21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs @@ -38,7 +38,10 @@ fn wrong_type(key: &str, expected: &str, value: &Bound<'_, PyAny>) -> PyErr { /// An authored string, preserving a present empty string. #[pyfunction(name = "get_config_str")] #[pyo3(signature = (*, values, key))] -fn config_str<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult>> { +pub(crate) fn config_str<'py>( + values: &Bound<'py, PyAny>, + key: &str, +) -> PyResult>> { let Some(value) = authored(values, key)? else { return Ok(None); }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index bec5690c43..d76eb0322f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -11,3 +11,4 @@ pub(crate) mod model_config_builder; pub(crate) mod model_validator; pub(crate) mod project_defaults; pub(crate) mod registration; +pub(crate) mod template_data; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs index d56dbf58c4..8f9293b80f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs @@ -3,18 +3,18 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::model_config::{ - column_migrations, config_errors, config_templates, config_values, described_models, - header_columns, model_config_builder, model_validator, + column_migrations, config_errors, config_values, described_models, header_columns, + model_config_builder, model_validator, template_data, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { column_migrations::register(module)?; config_errors::register(module)?; - config_templates::register(module)?; config_values::register(module)?; described_models::register(module)?; header_columns::register(module)?; model_config_builder::register(module)?; model_validator::register(module)?; + template_data::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs new file mode 100644 index 0000000000..28dedd70e0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs @@ -0,0 +1,213 @@ +//! Template expansion for compile inputs and effective vars, with the reads it makes recorded. + +use std::time::{SystemTime, UNIX_EPOCH}; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::scenario_command::main::format_captured_at::format_captured_at; +use sqlbuild_model_config::templates::models::TemplateOptions; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::model_config::config_templates::{ + TemplateOutcome, TemplateSources, expand_config_templates, lone_surrogate_site, + resolve_effective_vars, +}; + +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; +const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; +const CONFIG_ERROR_MODULE: &str = "sqlbuild.compiler.model_config.main._native_config_error"; +const UUID_MODULE: &str = "uuid"; +const ENVIRONMENT_READ: &str = "env"; +const RUN_ID_KEY: &str = "run.id"; +const RUN_TARGET_KEY: &str = "run.target"; +const MODEL_CONFIG_LABEL: &str = "model config"; +const RUN_ID_SUFFIX_LENGTH: usize = 12; +const TIMESTAMP_SEPARATORS: [char; 2] = ['-', ':']; + +/// Record native template environment and context reads for project reuse, in order. +pub(crate) fn record_reads<'a>( + py: Python<'_>, + reads: impl IntoIterator, +) -> PyResult<()> { + let registry: Bound<'_, PyAny> = imported(py, COMPILE_CONSTANTS_MODULE, "COMPILE_INPUT_READS")?; + for (kind, name) in reads { + let method: &str = if kind == ENVIRONMENT_READ { + "environment_read" + } else { + "context_read" + }; + registry.call_method1(method, (name,))?; + } + Ok(()) +} + +/// Record `outcome`'s reads, then return its value or raise its error as Python's stage does. +fn applied<'py>(py: Python<'py>, outcome: TemplateOutcome<'py>) -> PyResult> { + let (value, reads) = outcome; + record_reads(py, reads.iter().map(|(kind, name)| (*kind, name.as_str())))?; + match value { + Ok(value) => Ok(value), + Err(error) => Err(PyErr::from_value(call_with( + &imported(py, CONFIG_ERROR_MODULE, "native_config_error")?, + &[("error", error.into_bound(py).into_any())], + )?)), + } +} + +/// The process environment as compile text reads it. +fn environment(py: Python<'_>) -> PyResult> { + imported(py, UNICODE_ENVIRONMENT_MODULE, "UnicodeEnvironment")?.call0() +} + +/// `value` with every `${...}` template expanded, recording the reads it made. +pub(crate) fn expanded_template_data<'py>( + value: &Bound<'py, PyAny>, + (variables, context_values): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), + (options, label): (TemplateOptions, &str), +) -> PyResult> { + let py: Python<'py> = value.py(); + let sources = TemplateSources(variables.clone(), environment(py)?, context_values.clone()); + applied( + py, + expand_config_templates(py, value, sources, (options, label))?, + ) +} + +/// Merged effective vars with recursive `${name}` expansion. +pub(crate) fn expanded_effective_vars<'py>( + raw_values: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = raw_values.py(); + if let Some((name, path, byte_offset)) = lone_surrogate_site(raw_values)? { + let error: Bound<'py, PyAny> = imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")? + .call( + (format!( + "Variable '{name}' holds a lone surrogate in {path} at UTF-8 byte \ + {byte_offset}, which is not valid Unicode text" + ),), + Some(&keywords( + py, + &[( + "help", + PyString::new( + py, + &format!( + "Set '{name}' to valid Unicode text in --vars or the [vars] \ + table; the value is not shown because it may be a secret" + ), + ) + .into_any(), + )], + )?), + )?; + return Err(PyErr::from_value(error)); + } + applied( + py, + resolve_effective_vars(py, raw_values, environment(py)?)?, + ) +} + +/// Model header `values` with `${name}`, `${ENV:...}` and early `run.*` templates resolved. +pub(crate) fn early_model_templates<'py>( + values: &Bound<'py, PyAny>, + effective_vars: &Bound<'py, PyAny>, + (target_name, run_id): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = values.py(); + let context_values: Bound<'py, PyDict> = PyDict::new(py); + context_values.set_item(RUN_ID_KEY, run_id)?; + context_values.set_item(RUN_TARGET_KEY, target_name)?; + expanded_template_data( + values, + (effective_vars.downcast()?, &context_values), + ( + TemplateOptions { + allow_context: true, + preserve_context_tokens: false, + preserve_unknown_context: true, + }, + MODEL_CONFIG_LABEL, + ), + ) +} + +/// The selected run id, or a new `YYYYMMDDTHHMMSSZ_<12 hex>` compile invocation id. +pub(crate) fn run_id<'py>(selected_run_id: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = selected_run_id.py(); + if !selected_run_id.is_none() { + return Ok(selected_run_id.clone()); + } + let epoch_seconds: i64 = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_secs()).unwrap_or(i64::MAX) + }); + let timestamp: String = format_captured_at(epoch_seconds).replace(TIMESTAMP_SEPARATORS, ""); + let suffix: String = imported(py, UUID_MODULE, "uuid4")? + .call0()? + .getattr("hex")? + .extract()?; + let suffix: &str = suffix.get(..RUN_ID_SUFFIX_LENGTH).unwrap_or(&suffix); + Ok(PyString::new(py, &format!("{timestamp}_{suffix}")).into_any()) +} + +/// Recursively expand template strings inside supported Python container values. +#[pyfunction] +#[pyo3(signature = (**request))] +fn expand_template_data<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request: Request<'_, 'py> = Request::new("expand_template_data", request)?; + let flag = |name: &str| -> PyResult { request.get(name)?.is_truthy() }; + expanded_template_data( + &request.get("value")?, + ( + request.get("variables")?.downcast()?, + request.get("context_values")?.downcast()?, + ), + ( + TemplateOptions { + allow_context: flag("allow_context")?, + preserve_context_tokens: flag("preserve_context_tokens")?, + preserve_unknown_context: flag("preserve_unknown_context")?, + }, + &request.get("context_label")?.extract::()?, + ), + ) +} + +/// Resolve merged effective vars with recursive `${name}` expansion. +#[pyfunction] +fn expand_effective_vars<'py>(raw_values: &Bound<'py, PyDict>) -> PyResult> { + expanded_effective_vars(raw_values) +} + +/// Record native template environment and context reads for project reuse, in order. +#[pyfunction] +fn record_template_reads(py: Python<'_>, reads: Vec<(String, String)>) -> PyResult<()> { + record_reads( + py, + reads + .iter() + .map(|(kind, name)| (kind.as_str(), name.as_str())), + ) +} + +/// Resolve a stable compile invocation id. +#[pyfunction] +#[pyo3(signature = (*, selected_run_id))] +fn resolve_run_id<'py>(selected_run_id: &Bound<'py, PyAny>) -> PyResult> { + run_id(selected_run_id) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(expand_template_data, module)?)?; + module.add_function(wrap_pyfunction!(expand_effective_vars, module)?)?; + module.add_function(wrap_pyfunction!(record_template_reads, module)?)?; + module.add_function(wrap_pyfunction!(resolve_run_id, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs index 52abfd9a36..c5da29096c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs @@ -44,7 +44,7 @@ fn label_facts(entry: &Bound<'_, PyAny>) -> PyResult { }) } -fn optional_facts(entry: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn optional_facts(entry: &Bound<'_, PyAny>) -> PyResult> { if entry.is_none() { return Ok(None); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs new file mode 100644 index 0000000000..404fbe3599 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs @@ -0,0 +1,61 @@ +//! Per-model actions from each model's own change, and the incremental replacement predicate. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; + +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const INCREMENTAL_MATERIALIZATION: &str = "incremental"; +const CREATE_TABLE_ACTION: &str = "create_table"; + +/// Each selected model's change with the backfill its own `replay_on_change` decided. +#[pyfunction] +#[pyo3(signature = (*, scope, changes))] +fn resolve_model_actions<'py>( + scope: &Bound<'py, PyAny>, + changes: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = scope.py(); + let model_kind: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + let action_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "ResolvedModelAction")?; + let selected: Bound<'py, PyAny> = scope.getattr("selected_keys")?; + let models_by_name: Bound<'py, PyAny> = scope.getattr("models_by_name")?; + let model_changes: Bound<'py, PyAny> = changes.getattr("models")?; + let resolved: Bound<'py, PyDict> = PyDict::new(py); + for key in scope.getattr("execution_order")?.try_iter()? { + let key: Bound<'py, PyAny> = key?; + if !selected.contains(&key)? || !key.getattr("resource_type")?.eq(&model_kind)? { + continue; + } + let name: Bound<'py, PyAny> = key.getattr("name")?; + let change: Bound<'py, PyAny> = model_changes.call_method1("get", (&name,))?; + if !models_by_name.contains(&name)? || change.is_none() { + continue; + } + let backfill: Bound<'py, PyAny> = change.getattr("backfill")?; + resolved.set_item( + name, + call_with(&action_class, &[("change", change), ("backfill", backfill)])?, + )?; + } + call_with( + &imported(py, MODELS_MODULE, "PlannerResolvedActions")?, + &[("models", resolved.into_any())], + ) +} + +/// Incremental replacements, whose bounds must ignore the destination maximum. +#[pyfunction] +#[pyo3(signature = (*, materialization_type, action))] +fn replaces_incremental_relation(materialization_type: &str, action: &str) -> bool { + materialization_type == INCREMENTAL_MATERIALIZATION && action == CREATE_TABLE_ACTION +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_model_actions, module)?)?; + module.add_function(wrap_pyfunction!(replaces_incremental_relation, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs new file mode 100644 index 0000000000..19c22aa036 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -0,0 +1,820 @@ +//! Detect how each selected model and function changed since its recorded build. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_changes::constants::{ + CASE_FOLDING_DIALECT, EMPTY_FINGERPRINT_METADATA_JSON, +}; +use sqlbuild_analysis::planner_changes::main::detect_schema_changes::detect_schema_changes; +use sqlbuild_analysis::planner_changes::main::metadata_changes::metadata_changes; +use sqlbuild_analysis::planner_changes::main::model_backfill::model_backfill; +use sqlbuild_analysis::planner_changes::main::model_change_kind::model_change_kind; +use sqlbuild_analysis::planner_changes::main::recorded_declared_columns_hash::recorded_declared_columns_hash; +use sqlbuild_analysis::planner_changes::main::replay_policy_needed::replay_policy_needed; +use sqlbuild_analysis::planner_changes::main::resolve_replay_on_change::resolve_replay_on_change; +use sqlbuild_analysis::planner_changes::main::with_declared_columns_hash::with_declared_columns_hash; +use sqlbuild_analysis::planner_changes::models::{ + Backfill, BackfillAction, BackfillFacts, MetadataChanges, SchemaColumns, SchemaComparison, + SchemaFindingRow, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_changes::renames::origin_names; +use crate::bindings::_helpers::planner_changes::results::{ + MODELS_MODULE, backfill_result, change_kind, planner_enum, replay_policy_error, schema_findings, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const MODEL_METADATA_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.model_metadata"; +const VERSION_METADATA_MODULE: &str = + "sqlbuild.compiler.planner.main.identity.version_identity_model_metadata"; +const FUNCTION_HASHES_MODULE: &str = + "sqlbuild.compiler.planner.main.identity.version_identity_function_hashes"; +const HASHING_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.hashing"; +const FUNCTIONS_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.functions"; +const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; +const TYPES_EQUAL_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; +const DYNAMIC_FAMILIES_MODULE: &str = "sqlbuild._native"; +const DEBUG_EVENT_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; +const LOG_SQL_MODULE: &str = "sqlbuild.diagnostics.main.log_sql"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const CHANGES_LOGGER: &str = "sqlbuild.planner.changes"; +const REPLAY_CONFIG_KEY: &str = "replay_on_change"; + +/// Inputs shared by every model of one detection pass. +struct ModelPass<'py> { + snapshot: Bound<'py, PyAny>, + query_change_tracking: bool, + full_refresh: bool, + function_local_hashes: Bound<'py, PyAny>, + hook_version_hashes: Bound<'py, PyAny>, +} + +/// Selected model and function changes. +#[pyfunction] +#[pyo3(signature = (**options))] +fn detect_changes<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "detect_changes", + options, + ( + &["project", "scope", "snapshot", "full_refresh"], + &["expected_version_hashes", "expected_metadata_jsons"], + ), + )?; + let (project, scope, snapshot) = ( + arguments.value("project")?, + arguments.value("scope")?, + arguments.value("snapshot")?, + ); + let (project, scope, snapshot) = (&project, &scope, &snapshot); + let full_refresh: bool = arguments.flag("full_refresh")?; + let models: Bound<'py, PyDict> = scope_model_changes( + (project, scope, snapshot), + None, + ( + arguments.value("expected_version_hashes")?, + arguments.value("expected_metadata_jsons")?, + ), + )?; + let functions: Bound<'py, PyDict> = PyDict::new(py); + let result_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "FunctionChangeResult")?; + let fingerprint_sql_of: Bound<'py, PyAny> = imported( + py, + FUNCTIONS_MODULE, + "build_compiled_function_fingerprint_sql", + )?; + let detect_function: Bound<'py, PyAny> = + imported(py, FUNCTIONS_MODULE, "detect_function_change")?; + let selected: Bound<'py, PyAny> = scope.getattr("selected_keys")?; + for function in project.getattr("functions")?.try_iter()? { + let function: Bound<'py, PyAny> = function?; + let name: Bound<'py, PyAny> = function.getattr("name")?; + let fingerprint_sql: Bound<'py, PyAny> = fingerprint_sql_of.call1((&function,))?; + let mut arguments: Vec<(&str, Bound<'py, PyAny>)> = + vec![("fingerprint_sql", fingerprint_sql.clone())]; + if selected.contains(function.getattr("key")?)? { + let reason: Bound<'py, PyAny> = call_with( + &detect_function, + &[ + ("function", function.clone()), + ("fingerprint_sql", fingerprint_sql), + ( + "fingerprint", + snapshot + .getattr("fingerprints")? + .getattr("functions")? + .call_method1("get", (&name,))?, + ), + ( + "query_change_tracking", + project + .getattr("settings")? + .getattr("query_change_tracking")?, + ), + ( + "full_refresh", + full_refresh.into_pyobject(py)?.to_owned().into_any(), + ), + ("dialect", snapshot.getattr("column_dialect")?), + ], + )?; + arguments.push(("reason", reason)); + } + functions.set_item(name, call_with(&result_class, &arguments)?)?; + } + call_with( + &imported(py, MODELS_MODULE, "PlannerChangeResults")?, + &[ + ("models", models.into_any()), + ("functions", functions.into_any()), + ], + ) +} + +/// Selected model changes, without inspecting functions. +#[pyfunction] +#[pyo3(signature = (**options))] +fn detect_model_changes_in_scope<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "detect_model_changes_in_scope", + options, + ( + &["project", "scope", "snapshot"], + &[ + "query_change_tracking", + "expected_version_hashes", + "expected_metadata_jsons", + ], + ), + )?; + scope_model_changes( + ( + &arguments.value("project")?, + &arguments.value("scope")?, + &arguments.value("snapshot")?, + ), + arguments.value("query_change_tracking")?.extract()?, + ( + arguments.value("expected_version_hashes")?, + arguments.value("expected_metadata_jsons")?, + ), + ) +} + +/// One model's change and backfill. +#[pyfunction] +#[pyo3(signature = (**options))] +fn detect_model_changes<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "detect_model_changes", + options, + ( + &[ + "model", + "snapshot", + "sql_analysis_enabled", + "query_change_tracking", + "full_refresh", + ], + &[ + "function_local_hashes", + "hook_version_hashes", + "expected_version_hash", + "expected_metadata_json", + ], + ), + )?; + let pass = ModelPass { + snapshot: arguments.value("snapshot")?, + query_change_tracking: arguments.flag("query_change_tracking")?, + full_refresh: arguments.flag("full_refresh")?, + function_local_hashes: arguments.value("function_local_hashes")?, + hook_version_hashes: arguments.value("hook_version_hashes")?, + }; + model_change( + &pass, + &arguments.value("model")?, + ( + arguments.value("expected_version_hash")?.extract()?, + arguments.value("expected_metadata_json")?.extract()?, + ), + ) +} + +fn scope_model_changes<'py>( + (project, scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + query_change_tracking: Option, + (version_hashes, metadata_jsons): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = project.py(); + let tracking: bool = match query_change_tracking { + Some(value) => value, + None => project + .getattr("settings")? + .getattr("query_change_tracking")? + .extract()?, + }; + let function_local_hashes: Bound<'py, PyAny> = call_with( + &imported(py, FUNCTION_HASHES_MODULE, "build_function_local_hashes")?, + &[ + ("functions", project.getattr("functions")?), + ("dialect", project.getattr("sql_analysis_dialect")?), + ], + )?; + let hook_version_hashes: Bound<'py, PyDict> = PyDict::new(py); + let identities: Bound<'py, PyAny> = + imported(py, HOOK_IDENTITIES_MODULE, "build_hook_identities")? + .call1((project.getattr("hook_functions")?,))?; + for item in identities.call_method0("items")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + hook_version_hashes.set_item( + item.get_item(0)?, + item.get_item(1)?.getattr("version_hash")?, + )?; + } + let pass = ModelPass { + snapshot: snapshot.clone(), + query_change_tracking: tracking, + full_refresh: false, + function_local_hashes, + hook_version_hashes: hook_version_hashes.into_any(), + }; + let model_kind: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + let selected: Bound<'py, PyAny> = scope.getattr("selected_keys")?; + let models_by_name: Bound<'py, PyAny> = scope.getattr("models_by_name")?; + let changes: Bound<'py, PyDict> = PyDict::new(py); + for key in scope.getattr("execution_order")?.try_iter()? { + let key: Bound<'py, PyAny> = key?; + if !selected.contains(&key)? || !key.getattr("resource_type")?.eq(&model_kind)? { + continue; + } + let model: Bound<'py, PyAny> = + models_by_name.call_method1("get", (key.getattr("name")?,))?; + if model.is_none() { + continue; + } + let name: Bound<'py, PyAny> = model.getattr("name")?; + let expected: (Option, Option) = ( + lookup(&version_hashes, &name)?, + lookup(&metadata_jsons, &name)?, + ); + changes.set_item(&name, model_change(&pass, &model, expected)?)?; + } + Ok(changes) +} + +fn lookup(mapping: &Bound<'_, PyAny>, name: &Bound<'_, PyAny>) -> PyResult> { + if mapping.is_none() { + return Ok(None); + } + mapping.call_method1("get", (name,))?.extract() +} + +fn model_change<'py>( + pass: &ModelPass<'py>, + model: &Bound<'py, PyAny>, + (expected_version_hash, expected_metadata_json): (Option, Option), +) -> PyResult> { + let py: Python<'py> = model.py(); + let snapshot: &Bound<'py, PyAny> = &pass.snapshot; + let model_name: Bound<'py, PyAny> = model.getattr("name")?; + let columns_hash: String = call_with( + &imported(py, MODEL_METADATA_MODULE, "declared_columns_hash")?, + &[("model", model.clone())], + )? + .extract()?; + let base_metadata: String = match expected_metadata_json.filter(|text| !text.is_empty()) { + Some(text) => text, + None => call_with( + &imported( + py, + VERSION_METADATA_MODULE, + "build_model_version_identity_metadata_json", + )?, + &[ + ("model", model.clone()), + ("function_local_hashes", pass.function_local_hashes.clone()), + ("hook_version_hashes", pass.hook_version_hashes.clone()), + ], + )? + .extract()?, + }; + let metadata_json: String = with_declared_columns_hash(&base_metadata, &columns_hash); + let mut result = ChangeResult::new( + py, + &model_name, + &metadata_json, + expected_version_hash.as_deref(), + )?; + if pass.full_refresh { + return result.finish( + Backfill { + action: BackfillAction::Full, + duration: None, + }, + "no_change", + ); + } + let relation_exists: bool = snapshot + .getattr("existing_relations")? + .contains(&model_name)?; + let fingerprint: Bound<'py, PyAny> = snapshot + .getattr("fingerprints")? + .getattr("models")? + .call_method1("get", (&model_name,))?; + let recorded: Option = recorded_fingerprint(&fingerprint)?; + let dialect: Bound<'py, PyAny> = snapshot.getattr("column_dialect")?; + let compiled_query_hash: Option = if pass.query_change_tracking && recorded.is_some() { + Some( + call_with( + &imported(py, HASHING_MODULE, "model_definition_hash")?, + &[ + ("model_name", model_name.clone()), + ("query_sql", model.getattr("query_sql")?), + ("dialect", dialect.clone()), + ], + )? + .extract()?, + ) + } else { + None + }; + if let Some(recorded) = &recorded { + result.previous(recorded); + } + let definition_changed: bool = match (&compiled_query_hash, &recorded) { + (Some(hash), Some(recorded)) => *hash != recorded.definition_hash, + _ => false, + }; + if !relation_exists && snapshot.getattr("renamed_models")?.contains(&model_name)? { + result.query_changed = definition_changed; + let backfill: Backfill = if definition_changed { + replay_policy(model)? + } else { + Backfill { + action: BackfillAction::ForwardOnly, + duration: None, + } + }; + return result.finish(backfill, "renamed"); + } + if !relation_exists { + result.clear_previous(); + result.recorded_build_relation_missing = recorded.is_some(); + return result.finish( + Backfill { + action: BackfillAction::Full, + duration: None, + }, + "first_run", + ); + } + let recorded_metadata: Option<&str> = recorded + .as_ref() + .map(|recorded| recorded.metadata_json.as_str()) + .filter(|text| *text != EMPTY_FINGERPRINT_METADATA_JSON); + let renames: Bound<'py, PyAny> = snapshot.getattr("reference_renames")?; + let origins: Option> = match (&compiled_query_hash, &recorded) { + (Some(_), Some(recorded)) if definition_changed && renames.is_truthy()? => origin_names( + (model.getattr("query_sql")?.extract()?, &fingerprint), + &renames, + (&recorded.definition_hash, &dialect), + )?, + _ => None, + }; + let changes: MetadataChanges = metadata_changes( + (&metadata_json, recorded_metadata), + origins.as_deref(), + pass.query_change_tracking, + ); + let mut query_changed: bool = false; + if let (Some(hash), Some(recorded)) = (&compiled_query_hash, &recorded) { + query_changed = origins.is_none() && definition_changed; + log_query_check( + (model, &model_name, &fingerprint), + hash, + (&recorded.definition_hash, query_changed), + )?; + } + let findings: Vec = + model_schema_findings(model, snapshot, &model_name, &dialect)?; + let declared_columns_changed: bool = + recorded_declared_columns_hash(recorded_metadata).is_some_and(|hash| hash != columns_hash); + let facts = BackfillFacts { + query_changed, + functions_changed: !changes.changed_functions.is_empty(), + config_changed: changes.config_changed, + declared_columns_changed, + schema_changed: !findings.is_empty(), + }; + let replay: Option = if replay_policy_needed(facts) { + Some(replay_policy(model)?) + } else { + None + }; + let backfill: Backfill = model_backfill(facts, replay); + result.query_changed = query_changed; + result.config_changed = changes.config_changed; + result.findings = findings; + result.changed_functions = changes.changed_functions; + result.finish_kind(backfill, change_kind(py, model_change_kind(facts))?) +} + +/// The model's resolved `replay_on_change` policy. +fn replay_policy(model: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = model.py(); + let raw: Option = call_with( + &imported(py, NATIVE_MODULE, "get_config_str")?, + &[ + ("values", model.getattr("config")?.getattr("values")?), + ("key", PyString::new(py, REPLAY_CONFIG_KEY).into_any()), + ], + )? + .extract()?; + resolve_replay_on_change(raw.as_deref()).map_err(|error| replay_policy_error(py, error)) +} + +/// The recorded fingerprint fields change detection reads. +struct Recorded { + definition_hash: String, + metadata_json: String, + version_hash: String, +} + +fn recorded_fingerprint(fingerprint: &Bound<'_, PyAny>) -> PyResult> { + if fingerprint.is_none() { + return Ok(None); + } + Ok(Some(Recorded { + definition_hash: fingerprint.getattr("definition_hash")?.extract()?, + metadata_json: fingerprint.getattr("metadata_json")?.extract()?, + version_hash: fingerprint.getattr("version_hash")?.extract()?, + })) +} + +fn log_query_check( + (model, model_name, fingerprint): (&Bound<'_, PyAny>, &Bound<'_, PyAny>, &Bound<'_, PyAny>), + compiled_query_hash: &str, + (definition_hash, query_changed): (&str, bool), +) -> PyResult<()> { + let py: Python<'_> = model.py(); + let logger: Bound<'_, PyAny> = + imported(py, "logging", "getLogger")?.call1((CHANGES_LOGGER,))?; + let text = |value: &str| PyString::new(py, value).into_any(); + call_with( + &imported(py, DEBUG_EVENT_MODULE, "log_debug_event")?, + &[ + ("logger", logger.clone()), + ( + "message", + text(&format!( + "fingerprint comparison compiled_query_hash={compiled_query_hash} \ + fingerprint_definition_hash={definition_hash} query_changed={}", + if query_changed { "True" } else { "False" } + )), + ), + ("sqlbuild_subject", text("model")), + ("sqlbuild_name", model_name.clone()), + ("sqlbuild_event", text("query_change_check")), + ("sqlbuild_phase", text("planner")), + ( + "sqlbuild_status", + text(if query_changed { + "changed" + } else { + "unchanged" + }), + ), + ], + )?; + let log_sql: Bound<'_, PyAny> = imported(py, LOG_SQL_MODULE, "log_sql")?; + call_with( + &log_sql, + &[ + ("logger", logger.clone()), + ("sql", model.getattr("query_sql")?), + ("action", text("compiled_query")), + ], + )?; + call_with( + &log_sql, + &[ + ("logger", logger), + ("sql", fingerprint.getattr("definition")?), + ("action", text("fingerprint_definition")), + ], + )?; + Ok(()) +} + +fn model_schema_findings<'py>( + model: &Bound<'py, PyAny>, + snapshot: &Bound<'py, PyAny>, + model_name: &Bound<'py, PyAny>, + dialect: &Bound<'py, PyAny>, +) -> PyResult> { + let warehouse: Bound<'py, PyAny> = snapshot + .getattr("existing_columns")? + .call_method1("get", (model_name,))?; + if warehouse.is_none() { + return Ok(Vec::new()); + } + let schema_entry: Bound<'py, PyAny> = model.getattr("schema_entry")?; + let mut yml: Vec<(String, String)> = Vec::new(); + let mut type_enforcement: bool = false; + let mut dynamic: Option> = None; + if !schema_entry.is_none() { + for column in schema_entry.getattr("columns")?.try_iter()? { + let column: Bound<'py, PyAny> = column?; + if let Some(kind) = column.getattr("type")?.extract::>()? { + yml.push((column.getattr("name")?.extract()?, kind)); + } + } + type_enforcement = schema_entry + .getattr("type_enforcement")? + .extract::>()? + .unwrap_or(false); + let families: Bound<'py, PyAny> = schema_entry.getattr("dynamic_columns")?; + if families.is_truthy()? { + dynamic = Some(families); + } + } + let inferred: Option)>> = { + let columns: Bound<'py, PyAny> = model.getattr("inferred_columns")?; + if columns.is_none() { + None + } else { + let mut items: Vec<(String, Option)> = Vec::new(); + for column in columns.try_iter()? { + let column: Bound<'py, PyAny> = column?; + items.push(( + column.getattr("name")?.extract()?, + column.getattr("type")?.extract()?, + )); + } + Some(items) + } + }; + if yml.is_empty() && inferred.as_ref().is_none_or(Vec::is_empty) { + return Ok(Vec::new()); + } + let inferred_schema_complete: bool = !model.getattr("fast_lineage_has_star")?.is_truthy()?; + schema_findings_of( + SchemaColumns { + yml, + inferred, + warehouse: column_pairs(&warehouse)?, + }, + (type_enforcement, inferred_schema_complete), + (dynamic, dialect), + ) +} + +fn column_pairs(columns: &Bound<'_, PyAny>) -> PyResult> { + let mut pairs: Vec<(String, String)> = Vec::new(); + for column in columns.try_iter()? { + let column: Bound<'_, PyAny> = column?; + pairs.push(( + column.getattr("name")?.extract()?, + column.getattr("type")?.extract()?, + )); + } + Ok(pairs) +} + +fn schema_findings_of<'py>( + columns: SchemaColumns, + (type_enforcement, inferred_schema_complete): (bool, bool), + (dynamic, dialect): (Option>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = dialect.py(); + let dialect_name: Option = dialect.extract()?; + let types_equal_function: Bound<'py, PyAny> = imported(py, TYPES_EQUAL_MODULE, "types_equal")?; + let mut types_equal = |left: &str, right: &str| -> PyResult { + call_with( + &types_equal_function, + &[ + ("left", PyString::new(py, left).into_any()), + ("right", PyString::new(py, right).into_any()), + ("dialect", dialect.clone()), + ], + )? + .is_truthy() + }; + let matching_function: Bound<'py, PyAny> = + imported(py, DYNAMIC_FAMILIES_MODULE, "matching_dynamic_families")?; + let mut single_family_type = |name: &str| -> PyResult> { + let Some(families) = &dynamic else { + return Ok(None); + }; + let matching: Bound<'py, PyAny> = call_with( + &matching_function, + &[ + ("families", families.clone()), + ("column_name", PyString::new(py, name).into_any()), + ], + )?; + if matching.len()? != 1 { + return Ok(None); + } + matching.get_item(0)?.getattr("type")?.extract() + }; + detect_schema_changes( + columns, + SchemaComparison { + type_enforcement, + inferred_schema_complete, + case_folding: dialect_name.as_deref() == Some(CASE_FOLDING_DIALECT), + }, + (&mut types_equal, &mut single_family_type), + ) +} + +/// Schema findings between declared or inferred columns and warehouse columns. +#[pyfunction(name = "detect_schema_changes")] +#[pyo3(signature = (**options))] +fn schema_changes<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "detect_schema_changes", + options, + ( + &[ + "yml_columns", + "inferred_columns", + "warehouse_columns", + "type_enforcement", + "inferred_schema_complete", + ], + &["dynamic_columns", "dialect"], + ), + )?; + let yml_columns: Bound<'py, PyAny> = arguments.value("yml_columns")?; + let inferred_columns: Bound<'py, PyAny> = arguments.value("inferred_columns")?; + let warehouse_columns: Bound<'py, PyAny> = arguments.value("warehouse_columns")?; + let type_enforcement: bool = arguments.flag("type_enforcement")?; + let inferred_schema_complete: bool = arguments.flag("inferred_schema_complete")?; + let dynamic_columns: Bound<'py, PyAny> = arguments.value("dynamic_columns")?; + let dialect: Bound<'py, PyAny> = arguments.value("dialect")?; + let py: Python<'py> = yml_columns.py(); + let inferred: Option)>> = if inferred_columns.is_none() { + None + } else { + let mut items: Vec<(String, Option)> = Vec::new(); + for column in inferred_columns.try_iter()? { + let column: Bound<'py, PyAny> = column?; + items.push(( + column.getattr("name")?.extract()?, + column.getattr("type")?.extract()?, + )); + } + Some(items) + }; + let dynamic: Option> = if dynamic_columns.is_truthy()? { + Some(dynamic_columns) + } else { + None + }; + let findings: Vec = schema_findings_of( + SchemaColumns { + yml: column_pairs(&yml_columns)?, + inferred, + warehouse: column_pairs(&warehouse_columns)?, + }, + (type_enforcement, inferred_schema_complete), + (dynamic, &dialect), + )?; + schema_findings(py, &findings) +} + +/// The `ChangeDetectionResult` being assembled for one model. +struct ChangeResult<'py> { + py: Python<'py>, + model_name: Bound<'py, PyAny>, + metadata_json: String, + version_hash: Option, + previous_metadata_json: Option, + previous_version_hash: Option, + query_changed: bool, + config_changed: bool, + recorded_build_relation_missing: bool, + findings: Vec, + changed_functions: Vec, +} + +impl<'py> ChangeResult<'py> { + fn new( + py: Python<'py>, + model_name: &Bound<'py, PyAny>, + metadata_json: &str, + version_hash: Option<&str>, + ) -> PyResult { + Ok(Self { + py, + model_name: model_name.clone(), + metadata_json: metadata_json.to_owned(), + version_hash: version_hash.map(str::to_owned), + previous_metadata_json: None, + previous_version_hash: None, + query_changed: false, + config_changed: false, + recorded_build_relation_missing: false, + findings: Vec::new(), + changed_functions: Vec::new(), + }) + } + + fn previous(&mut self, recorded: &Recorded) { + self.previous_metadata_json = Some(recorded.metadata_json.clone()); + self.previous_version_hash = Some(recorded.version_hash.clone()); + } + + fn clear_previous(&mut self) { + self.previous_metadata_json = None; + self.previous_version_hash = None; + } + + fn finish(self, backfill: Backfill, kind: &str) -> PyResult> { + let kind: Bound<'py, PyAny> = planner_enum(self.py, "ChangeKind", kind)?; + self.finish_kind(backfill, kind) + } + + fn finish_kind( + self, + backfill: Backfill, + kind: Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = self.py; + let flag = |value: bool| { + value + .into_pyobject(py) + .map(|flag| flag.to_owned().into_any()) + }; + call_with( + &imported(py, MODELS_MODULE, "ChangeDetectionResult")?, + &[ + ("model_name", self.model_name), + ("change_kind", kind), + ("query_changed", flag(self.query_changed)?), + ("config_changed", flag(self.config_changed)?), + ( + "fingerprint_metadata_json", + self.metadata_json.into_pyobject(py)?.into_any(), + ), + ( + "previous_metadata_json", + self.previous_metadata_json.into_pyobject(py)?.into_any(), + ), + ( + "fingerprint_version_hash", + self.version_hash.into_pyobject(py)?.into_any(), + ), + ( + "previous_version_hash", + self.previous_version_hash.into_pyobject(py)?.into_any(), + ), + ( + "schema_findings", + schema_findings(py, &self.findings)?.into_any(), + ), + ("backfill", backfill_result(py, &backfill)?), + ( + "recorded_build_relation_missing", + flag(self.recorded_build_relation_missing)?, + ), + ( + "changed_functions", + pyo3::types::PyTuple::new(py, self.changed_functions)?.into_any(), + ), + ], + ) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(detect_changes, module)?)?; + module.add_function(wrap_pyfunction!(detect_model_changes_in_scope, module)?)?; + module.add_function(wrap_pyfunction!(detect_model_changes, module)?)?; + module.add_function(wrap_pyfunction!(schema_changes, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs new file mode 100644 index 0000000000..e760bb22da --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs @@ -0,0 +1,46 @@ +//! Version-identity metadata helpers the remaining Python planner reads. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_changes::main::version_identity_payload::version_identity_payload; +use sqlbuild_analysis::planner_changes::main::with_declared_columns_hash::with_declared_columns_hash as with_hash; +use sqlbuild_analysis::planner_changes::main::without_declared_columns_hash::without_declared_columns_hash as without_hash; +use sqlbuild_analysis::planner_changes::models::IdentityPayload; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +/// The decoded metadata that participates in version identity, the raw text when it is not JSON, or `None`. +#[pyfunction] +#[pyo3(signature = (metadata_json))] +fn version_identity_metadata_payload<'py>( + py: Python<'py>, + metadata_json: Option<&str>, +) -> PyResult> { + match version_identity_payload(metadata_json, false) { + IdentityPayload::Absent => Ok(py.None().into_bound(py)), + IdentityPayload::Raw(text) => Ok(text.into_pyobject(py)?.into_any()), + IdentityPayload::Object(object) => { + imported(py, "json", "loads")?.call1((serde_json::Value::Object(object).to_string(),)) + } + } +} + +/// Fingerprint metadata JSON that also records the model's declared-columns hash. +#[pyfunction] +#[pyo3(signature = (*, metadata_json, declared_columns_hash))] +fn with_declared_columns_hash(metadata_json: &str, declared_columns_hash: &str) -> String { + with_hash(metadata_json, declared_columns_hash) +} + +/// Fingerprint metadata JSON without the declared-columns hash key, if present. +#[pyfunction] +fn without_declared_columns_hash(metadata_json: &str) -> String { + without_hash(metadata_json) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(version_identity_metadata_payload, module)?)?; + module.add_function(wrap_pyfunction!(with_declared_columns_hash, module)?)?; + module.add_function(wrap_pyfunction!(without_declared_columns_hash, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/mod.rs new file mode 100644 index 0000000000..301a6f231c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod actions; +pub(crate) mod detect; +pub(crate) mod metadata; +pub(crate) mod registration; +pub(crate) mod renames; +pub(crate) mod results; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/registration.rs new file mode 100644 index 0000000000..750b787720 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/registration.rs @@ -0,0 +1,12 @@ +//! Register planner change detection. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::planner_changes::{actions, detect, metadata, renames}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + actions::register(module)?; + detect::register(module)?; + metadata::register(module)?; + renames::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs new file mode 100644 index 0000000000..d19d59345b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs @@ -0,0 +1,275 @@ +//! Model renames that downstream `__ref` names may have been rewritten for. + +use std::collections::BTreeSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_changes::main::origin_reference_names::origin_reference_names; +use sqlbuild_analysis::planner_changes::models::{NameSpan, RenameCandidate}; +use sqlbuild_rules::lint_text::main::neutralize_interpolation::neutralize_interpolation; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const HASHING_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.hashing"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; +const GENERIC_DIALECT: &str = "generic"; +const FINAL_NEWLINE: &str = "\n"; + +const REF_CALL_PREFIX: &str = "__ref"; + +/// The rename mapping under which the model query hashes to its recorded definition. +pub(crate) fn origin_names( + (query_sql, fingerprint): (String, &Bound<'_, PyAny>), + renames: &Bound<'_, PyAny>, + (recorded_hash, dialect): (&str, &Bound<'_, PyAny>), +) -> PyResult>> { + let py: Python<'_> = fingerprint.py(); + let recorded_at: Bound<'_, PyAny> = utc(&fingerprint.getattr("ts")?)?; + let mut candidates: Vec = Vec::new(); + for rename in renames.try_iter()? { + let rename: Bound<'_, PyAny> = rename?; + let renamed_at: Bound<'_, PyAny> = rename.getattr("recorded_at")?; + candidates.push(RenameCandidate { + new_name: rename.getattr("new_name")?.extract()?, + origin_name: rename.getattr("origin_name")?.extract()?, + applies: renamed_at.is_none() || recorded_at.lt(utc(&renamed_at)?)?, + }); + } + let dialect_name: String = dialect + .extract::>()? + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| GENERIC_DIALECT.to_owned()); + let mut spans = |names: &BTreeSet| ref_name_spans(&query_sql, &dialect_name, names); + let compute_hash: Bound<'_, PyAny> = imported(py, NATIVE_MODULE, "compute_query_hash")?; + let mut query_hash = |text: &str| -> PyResult { + call_with( + &compute_hash, + &[ + ("query_sql", PyString::new(py, text).into_any()), + ("dialect", dialect.clone()), + ], + )? + .extract() + }; + origin_reference_names( + (&query_sql, recorded_hash), + &candidates, + (&mut spans, &mut query_hash), + ) +} + +fn ref_name_spans( + query_sql: &str, + dialect: &str, + names: &BTreeSet, +) -> PyResult> { + let neutralized = neutralize_interpolation(query_sql, dialect, false).map_err(value_error)?; + let mut spans: Vec = Vec::new(); + for site in neutralized.sites { + let Some((offset, name)) = ref_call_name(&site.original_text) else { + continue; + }; + if !names.contains(name) { + continue; + } + let start: usize = site.original_start + site.original_text[..offset].chars().count(); + spans.push(NameSpan { + start, + end: start + name.chars().count(), + name: name.to_owned(), + }); + } + Ok(spans) +} + +/// The byte offset and name of a whole `__ref('name')` or `__ref("name")` call. +fn ref_call_name(text: &str) -> Option<(usize, &str)> { + let rest: &str = text.strip_prefix(REF_CALL_PREFIX)?.trim_start(); + let rest: &str = rest.strip_prefix('(')?.trim_start(); + let quote: char = rest + .chars() + .next() + .filter(|quote| matches!(quote, '\'' | '"'))?; + let body: &str = &rest[1..]; + let length: usize = body.find(['\'', '"'])?; + let name: &str = &body[..length]; + let tail: &str = body[length..].strip_prefix(quote)?.trim_start(); + let tail: &str = tail.strip_prefix(')')?; + if name.is_empty() || !(tail.is_empty() || tail == FINAL_NEWLINE) { + return None; + } + Some((text.len() - body.len(), name)) +} + +/// `value` in UTC; a naive value is taken as UTC. +fn utc<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + let zone: Bound<'py, PyAny> = imported(py, "datetime", "UTC")?; + if value.getattr("tzinfo")?.is_none() || value.call_method0("utcoffset")?.is_none() { + return call_with(&value.getattr("replace")?, &[("tzinfo", zone)]); + } + value.call_method1("astimezone", (zone,)) +} + +/// The rename mapping under which `query_sql` hashes to the recorded definition, or `None`. +#[pyfunction(name = "origin_reference_names")] +#[pyo3(signature = (*, query_sql, recorded, renames, dialect))] +fn origin_reference_names_of<'py>( + query_sql: String, + recorded: &Bound<'py, PyAny>, + renames: &Bound<'py, PyAny>, + dialect: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = recorded.py(); + let recorded_hash: String = recorded.getattr("definition_hash")?.extract()?; + let Some(mapping) = origin_names((query_sql, recorded), renames, (&recorded_hash, dialect))? + else { + return Ok(None); + }; + let result: Bound<'py, PyDict> = PyDict::new(py); + for (name, origin) in mapping { + result.set_item(name, origin)?; + } + Ok(Some(result)) +} + +/// This plan's renames plus recorded renames, read only when a selected query that references models changed. +#[pyfunction] +#[pyo3(signature = (**options))] +fn plan_reference_renames<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "plan_reference_renames", + options, + ( + &[ + "runtime", "scope", "snapshot", "state", "schemas", "planned", + ], + &[], + ), + )?; + let (runtime, scope, snapshot) = ( + arguments.value("runtime")?, + arguments.value("scope")?, + arguments.value("snapshot")?, + ); + let (state, schemas, planned) = ( + arguments.value("state")?, + arguments.value("schemas")?, + arguments.value("planned")?, + ); + let (runtime, scope, snapshot) = (&runtime, &scope, &snapshot); + if !changed_referencing_query(scope, snapshot)? { + return Ok(PyTuple::empty(py)); + } + let state_schemas: Bound<'py, PyAny> = snapshot.getattr("migration_state_schemas")?; + let inspected: Bound<'py, PySet> = PySet::empty(py)?; + for schema in schemas.try_iter()? { + let schema: Bound<'py, PyAny> = schema?; + if state_schemas.contains(schema.call_method0("lower")?)? { + inspected.add(schema)?; + } + } + call_with( + &state.getattr("inspect_migration_events")?, + &[("schemas", inspected.into_any())], + )?; + let target_name: Bound<'py, PyAny> = runtime + .getattr("project")? + .getattr("effective_target_name")?; + let rename_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "ReferenceRename")?; + let mut renames: Vec> = Vec::new(); + for item in planned.call_method0("items")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + let (new_name, origin_name) = (item.get_item(0)?, item.get_item(1)?); + if new_name.ne(&origin_name)? { + renames.push(call_with( + &rename_class, + &[("new_name", new_name), ("origin_name", origin_name)], + )?); + } + } + for event in state.getattr("events")?.try_iter()? { + let event: Bound<'py, PyAny> = event?; + let event_target: Bound<'py, PyAny> = event.getattr("target_name")?; + let origin: Bound<'py, PyAny> = event.getattr("origin_model")?; + let destination: Bound<'py, PyAny> = event.getattr("destination_model")?; + if (event_target.is_none() || event_target.eq(&target_name)?) + && !origin.is_none() + && origin.ne(&destination)? + { + renames.push(call_with( + &rename_class, + &[ + ("new_name", destination), + ("origin_name", origin), + ("recorded_at", event.getattr("created_at")?), + ], + )?); + } + } + PyTuple::new(py, renames) +} + +fn changed_referencing_query( + scope: &Bound<'_, PyAny>, + snapshot: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = scope.py(); + let models_by_name: Bound<'_, PyAny> = scope.getattr("models_by_name")?; + let mut selected: Vec> = Vec::new(); + for key in scope.getattr("selected_keys")?.try_iter()? { + let model: Bound<'_, PyAny> = + models_by_name.call_method1("get", (key?.getattr("name")?,))?; + if !model.is_none() { + selected.push(model); + } + } + let recorded_models: Bound<'_, PyAny> = snapshot.getattr("fingerprints")?.getattr("models")?; + let ref_kind: Bound<'_, PyAny> = + imported(py, REFERENCE_TYPES_MODULE, "SqlReferenceKind")?.getattr("REF")?; + let definition_hash: Bound<'_, PyAny> = imported(py, HASHING_MODULE, "model_definition_hash")?; + for model in selected { + let recorded: Bound<'_, PyAny> = + recorded_models.call_method1("get", (model.getattr("name")?,))?; + if recorded.is_none() { + continue; + } + let mut references_models: bool = false; + for reference in model.getattr("references")?.try_iter()? { + if reference?.getattr("ref_kind")?.eq(&ref_kind)? { + references_models = true; + break; + } + } + if references_models + && call_with( + &definition_hash, + &[ + ("model_name", model.getattr("name")?), + ("query_sql", model.getattr("query_sql")?), + ("dialect", snapshot.getattr("column_dialect")?), + ], + )? + .ne(recorded.getattr("definition_hash")?)? + { + return Ok(true); + } + } + Ok(false) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(plan_reference_renames, module)?)?; + module.add_function(wrap_pyfunction!(origin_reference_names_of, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs new file mode 100644 index 0000000000..113cab7398 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs @@ -0,0 +1,109 @@ +//! The planner's Python change-detection result models. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use pyo3::{IntoPyObject, PyErr}; + +use sqlbuild_analysis::planner_changes::errors::UnknownReplayPolicy; +use sqlbuild_analysis::planner_changes::models::{ + Backfill, BackfillAction, ChangeKind, SchemaChangeKind, SchemaColumnSource, SchemaFindingRow, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +pub(crate) const MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +pub(crate) const TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; + +/// `PlannerInputError(message)` for an unknown replay policy. +pub(crate) fn replay_policy_error(py: Python<'_>, error: UnknownReplayPolicy) -> PyErr { + match imported(py, EXCEPTIONS_MODULE, "PlannerInputError") + .and_then(|class| class.call1((error.message(),))) + { + Ok(value) => PyErr::from_value(value), + Err(error) => error, + } +} + +/// `TYPES_MODULE.(value)`. +pub(crate) fn planner_enum<'py>( + py: Python<'py>, + name: &str, + value: &str, +) -> PyResult> { + imported(py, TYPES_MODULE, name)?.call1((value,)) +} + +/// `BackfillResult(action=..., duration=...)`. +pub(crate) fn backfill_result<'py>( + py: Python<'py>, + backfill: &Backfill, +) -> PyResult> { + let action: &str = match backfill.action { + BackfillAction::Full => "full", + BackfillAction::Bounded => "bounded", + BackfillAction::ForwardOnly => "forward", + }; + call_with( + &imported(py, MODELS_MODULE, "BackfillResult")?, + &[ + ("action", planner_enum(py, "BackfillAction", action)?), + ( + "duration", + backfill.duration.clone().into_pyobject(py)?.into_any(), + ), + ], + ) +} + +/// The `ChangeKind` member of a decided change. +pub(crate) fn change_kind<'py>(py: Python<'py>, kind: ChangeKind) -> PyResult> { + let value: &str = match kind { + ChangeKind::QueryChanged => "query_changed", + ChangeKind::FunctionChanged => "function_changed", + ChangeKind::SchemaChanged => "schema_changed", + ChangeKind::ConfigChanged => "config_changed", + ChangeKind::NoChange => "no_change", + }; + planner_enum(py, "ChangeKind", value) +} + +/// A tuple of `SchemaFinding` values. +pub(crate) fn schema_findings<'py>( + py: Python<'py>, + findings: &[SchemaFindingRow], +) -> PyResult> { + let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "SchemaFinding")?; + let mut items: Vec> = Vec::with_capacity(findings.len()); + for finding in findings { + let kind: &str = match finding.kind { + SchemaChangeKind::ColumnAdded => "column_added", + SchemaChangeKind::ColumnRemoved => "column_removed", + SchemaChangeKind::ColumnTypeChanged => "column_type_changed", + }; + let source: &str = match finding.source { + SchemaColumnSource::Yml => "yml", + SchemaColumnSource::SqlAnalysis => "sql_analysis", + }; + items.push(call_with( + &class, + &[ + ("kind", planner_enum(py, "SchemaChangeKind", kind)?), + ( + "column_name", + finding.column_name.clone().into_pyobject(py)?.into_any(), + ), + ("source", planner_enum(py, "SchemaColumnSource", source)?), + ( + "expected_type", + finding.expected_type.clone().into_pyobject(py)?.into_any(), + ), + ( + "actual_type", + finding.actual_type.clone().into_pyobject(py)?.into_any(), + ), + ], + )?); + } + PyTuple::new(py, items) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs new file mode 100644 index 0000000000..fbf692a524 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs @@ -0,0 +1,161 @@ +//! Audit attachment, target lineage and run scope. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PySet, PySetMethods, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_strategy::constants::{MODEL_REFERENCE_KIND, SOURCE_REFERENCE_KIND}; +use sqlbuild_analysis::planner_strategy::main::effective_run_scope::effective_run_scope; +use sqlbuild_analysis::planner_strategy::main::singular_attachment::singular_attachment; +use sqlbuild_analysis::planner_strategy::models::{AuditAttachmentKind, AuditRunScope}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::graph::algorithms::closure; +use crate::bindings::_helpers::planner_strategy::values::{ + PLANNER_EXCEPTIONS_MODULE, input_error, optional_string, text, +}; + +const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const MODEL_RESOURCE_TYPE: &str = "model"; +const FINAL_RUN_SCOPE: &str = "final"; + +/// `CompiledObjectKey(resource_type=CompiledResourceType(kind), name=name)`. +fn object_key<'py>(py: Python<'py>, kind: &str, name: &str) -> PyResult> { + call_with( + &imported(py, COMPILE_MODELS_MODULE, "CompiledObjectKey")?, + &[ + ( + "resource_type", + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.call1((kind,))?, + ), + ("name", PyString::new(py, name).into_any()), + ], + ) +} + +/// The names of an audit's references of one kind, in order. +fn reference_names(audit: &Bound<'_, PyAny>, kind: &str) -> PyResult> { + let mut names: Vec = Vec::new(); + for reference in audit.getattr("references")?.try_iter()? { + let reference: Bound<'_, PyAny> = reference?; + if text(&reference.getattr("ref_kind")?)? == kind { + names.push(text(&reference.getattr("ref_name")?)?); + } + } + Ok(names) +} + +fn attachment_value(kind: AuditAttachmentKind) -> &'static str { + match kind { + AuditAttachmentKind::Source => "source", + AuditAttachmentKind::Seed => "seed", + AuditAttachmentKind::Model => "model", + AuditAttachmentKind::End => "end", + } +} + +/// Audit attachment kind and attached target name; attached audits gate their target. +#[pyfunction] +#[pyo3(signature = (*, audit, downstream_deps))] +fn resolve_attachment_kind<'py>( + audit: &Bound<'py, PyAny>, + downstream_deps: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = audit.py(); + let attached_kind: Bound<'py, PyAny> = audit.getattr("attached_target_kind")?; + let (kind, name): (String, Option) = if attached_kind.is_none() { + let reaches = |from: &str, to: &str| -> PyResult { + let start: Bound<'py, PyAny> = object_key(py, MODEL_RESOURCE_TYPE, from)?; + closure(vec![start], downstream_deps, (false, None))?.contains(object_key( + py, + MODEL_RESOURCE_TYPE, + to, + )?) + }; + let (kind, name) = singular_attachment( + &reference_names(audit, MODEL_REFERENCE_KIND)?, + &reference_names(audit, SOURCE_REFERENCE_KIND)?, + &reaches, + )?; + (attachment_value(kind).to_owned(), name) + } else { + let name: Bound<'py, PyAny> = audit.getattr("attached_target_name")?; + if name.is_none() { + return Err(input_error( + py, + (PLANNER_EXCEPTIONS_MODULE, "PlannerInputError"), + &format!( + "audit '{}': attached audit is missing an attached target name", + text(&audit.getattr("name")?)? + ), + None, + )); + } + let kind: String = text( + &imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")? + .call1((attached_kind,))?, + )?; + (kind, Some(text(&name)?)) + }; + PyTuple::new( + py, + [ + imported(py, AUDITING_TYPES_MODULE, "AuditAttachmentKind")?.call1((kind,))?, + name.into_pyobject(py)?.into_any(), + ], + ) +} + +/// Whether an attached audit reads anything besides its target and the target's upstream. +#[pyfunction] +#[pyo3(signature = (*, audit, upstream_deps))] +fn reads_outside_target_lineage( + audit: &Bound<'_, PyAny>, + upstream_deps: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = audit.py(); + let kind: Bound<'_, PyAny> = audit.getattr("attached_target_kind")?; + let name: Bound<'_, PyAny> = audit.getattr("attached_target_name")?; + if kind.is_none() || name.is_none() { + return Ok(false); + } + let kind: String = + text(&imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind,))?)?; + let target: Bound<'_, PyAny> = object_key(py, &kind, &text(&name)?)?; + let lineage: Bound<'_, PySet> = closure(vec![target], upstream_deps, (true, None))?; + for dependency in audit.getattr("scope_deps")?.try_iter()? { + if !lineage.contains(dependency?)? { + return Ok(true); + } + } + Ok(false) +} + +/// The run scope after degradation: delta-and-final only for incremental and snapshot targets. +#[pyfunction] +#[pyo3(signature = (*, requested_run_scope, attached_model_materialization))] +fn resolve_effective_run_scope<'py>( + requested_run_scope: &Bound<'py, PyAny>, + attached_model_materialization: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = requested_run_scope.py(); + let requested: AuditRunScope = if text(requested_run_scope)? == FINAL_RUN_SCOPE { + AuditRunScope::Final + } else { + AuditRunScope::DeltaAndFinal + }; + let materialization: Option = optional_string(attached_model_materialization)?; + let scope: &str = match effective_run_scope(requested, materialization.as_deref()) { + AuditRunScope::Final => FINAL_RUN_SCOPE, + AuditRunScope::DeltaAndFinal => "delta_and_final", + }; + imported(py, AUDITING_TYPES_MODULE, "AuditRunScope")?.call1((scope,)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_attachment_kind, module)?)?; + module.add_function(wrap_pyfunction!(reads_outside_target_lineage, module)?)?; + module.add_function(wrap_pyfunction!(resolve_effective_run_scope, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs new file mode 100644 index 0000000000..848cb748f2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs @@ -0,0 +1,117 @@ +//! Warehouse cursor column type checks. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_strategy::errors::TypeParseFailure; +use sqlbuild_analysis::planner_strategy::main::classify_cursor_sql_type::classify_cursor_sql_type as classify; +use sqlbuild_analysis::planner_strategy::main::cursor_type_warning::cursor_type_warning; +use sqlbuild_analysis::planner_strategy::models::{CursorType, CursorTypeCheck}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_changes::results::planner_enum; +use crate::bindings::_helpers::planner_strategy::values::{optional_string, plan_warning, text}; + +const DEBUG_EVENT_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; +const PLANNER_LOGGER: &str = "sqlbuild.planner"; + +/// Log the Polyglot parse error that left a warehouse type unclassified. +fn log_parse_failure(py: Python<'_>, failure: &TypeParseFailure) -> PyResult<()> { + let text = |value: &str| PyString::new(py, value).into_any(); + call_with( + &imported(py, DEBUG_EVENT_MODULE, "log_debug_event")?, + &[ + ( + "logger", + imported(py, "logging", "getLogger")?.call1((PLANNER_LOGGER,))?, + ), + ( + "message", + text("cursor type classification parse failed; falling back"), + ), + ("sqlbuild_warehouse_type", text(&failure.normalized_type)), + ("sqlbuild_error", text(&failure.message)), + ], + ) + .map(|_| ()) +} + +/// The `CursorType` a supported cross-adapter SQL type holds, or `None`. +#[pyfunction] +#[pyo3(signature = (sql_type))] +fn classify_cursor_sql_type<'py>( + py: Python<'py>, + sql_type: &str, +) -> PyResult>> { + let detected: Option = match classify(sql_type) { + Ok(detected) => detected, + Err(failure) => { + log_parse_failure(py, &failure)?; + None + } + }; + detected + .map(|cursor_type| { + let value: &str = match cursor_type { + CursorType::Timestamp => "timestamp", + CursorType::Integer => "integer", + }; + planner_enum(py, "CursorType", value) + }) + .transpose() +} + +/// A `PlanWarning` when the warehouse cursor column type contradicts `cursor_type`. +#[pyfunction] +#[pyo3(signature = (**options))] +fn check_cursor_type_consistency<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult>> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "check_cursor_type_consistency", + options, + ( + &[ + "model_name", + "cursor_column", + "cursor_type", + "warehouse_columns", + "sql_analysis_enabled", + ], + &[], + ), + )?; + let model_name: Bound<'py, PyAny> = arguments.value("model_name")?; + let mut warehouse_columns: Vec<(String, String)> = Vec::new(); + for column in arguments.value("warehouse_columns")?.try_iter()? { + let column: Bound<'py, PyAny> = column?; + warehouse_columns.push(( + text(&column.getattr("name")?)?, + text(&column.getattr("type")?)?, + )); + } + let check = CursorTypeCheck { + model_name: text(&model_name)?, + cursor_column: optional_string(&arguments.value("cursor_column")?)?, + cursor_type: optional_string(&arguments.value("cursor_type")?)?, + warehouse_columns, + sql_analysis_enabled: arguments.flag("sql_analysis_enabled")?, + }; + let (warning, parse_failure) = cursor_type_warning(&check); + if let Some(failure) = parse_failure { + log_parse_failure(py, &failure)?; + } + warning + .map(|row| plan_warning(&model_name, row)) + .transpose() +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(classify_cursor_sql_type, module)?)?; + module.add_function(wrap_pyfunction!(check_cursor_type_consistency, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs new file mode 100644 index 0000000000..47bd1cb509 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs @@ -0,0 +1,172 @@ +//! Watermark inputs resolved through SQL reference lineage, and capped producers behind them. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_strategy::errors::LineageFailure; +use sqlbuild_analysis::planner_strategy::main::capped_watermark_input::capped_watermark_input; +use sqlbuild_analysis::planner_strategy::main::lineage_reference::lineage_reference; +use sqlbuild_analysis::planner_strategy::models::{ProducerFacts, ReferenceIdentity}; +use sqlbuild_analysis::planner_strategy::types::ReferenceGraph; + +use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; +use crate::bindings::_helpers::model_config::config_values::config_str; +use crate::bindings::_helpers::planner_strategy::microbatches::{ + CAP_ACTIONS, microbatch_limit_config, +}; +use crate::bindings::_helpers::planner_strategy::values::{ + COMPILE_EXCEPTIONS_MODULE, PLANNER_EXCEPTIONS_MODULE, input_error, text, +}; + +const MICROBATCH_MODE: &str = "microbatch"; + +/// The project's models and SQL functions by name. +struct ProjectLineage<'py> { + models_by_name: Bound<'py, PyAny>, + functions_by_name: Bound<'py, PyAny>, +} + +/// The references of the value under `name`, when the mapping has it. +fn mapped_references<'py>( + mapping: &Bound<'py, PyAny>, + name: &str, +) -> PyResult>>> { + let value: Bound<'py, PyAny> = mapping.call_method1("get", (name,))?; + if value.is_none() { + return Ok(None); + } + references(&value).map(Some) +} + +fn references<'py>(owner: &Bound<'py, PyAny>) -> PyResult>> { + owner.getattr("references")?.try_iter()?.collect() +} + +/// Whether a model's `incremental_mode` is microbatch. +fn microbatch_flag(model: &Bound<'_, PyAny>) -> PyResult { + let values: Bound<'_, PyAny> = model.getattr("config")?.getattr("values")?; + Ok(match config_str(&values, "incremental_mode")? { + Some(mode) => mode.eq(MICROBATCH_MODE)?, + None => false, + }) +} + +/// A model's watermark input names. +fn watermark_names(model: &Bound<'_, PyAny>) -> PyResult> { + resolve_cursor_input_roles(model)? + .getattr("watermark_inputs")? + .try_iter()? + .map(|name| text(&name?)) + .collect() +} + +impl<'py> ReferenceGraph for ProjectLineage<'py> { + type Reference = Bound<'py, PyAny>; + type Error = PyErr; + + fn identity(&self, reference: &Bound<'py, PyAny>) -> PyResult { + Ok(ReferenceIdentity { + kind: text(&reference.getattr("ref_kind")?)?, + name: text(&reference.getattr("ref_name")?)?, + }) + } + + fn model_references(&self, name: &str) -> PyResult>>> { + mapped_references(&self.models_by_name, name) + } + + fn function_references(&self, name: &str) -> PyResult>>> { + mapped_references(&self.functions_by_name, name) + } + + fn producer_facts(&self, name: &str) -> PyResult { + let producer: Bound<'py, PyAny> = self.models_by_name.get_item(name)?; + let values: Bound<'py, PyAny> = producer.getattr("config")?.getattr("values")?; + let capped: bool = match microbatch_limit_config(&values)?.1 { + Some(action) => CAP_ACTIONS.contains(&text(&action)?.as_str()), + None => false, + }; + Ok(ProducerFacts { + microbatch: microbatch_flag(&producer)?, + capped, + }) + } + + fn watermark_inputs(&self, name: &str) -> PyResult> { + watermark_names(&self.models_by_name.get_item(name)?) + } +} + +/// The Python error of a failed lineage walk. +fn lineage_error(py: Python<'_>, failure: LineageFailure) -> PyErr { + match failure { + LineageFailure::Graph(error) => error, + LineageFailure::NotInLineage { model, input } => input_error( + py, + (PLANNER_EXCEPTIONS_MODULE, "PlannerInputError"), + &LineageFailure::::message(&model, &input), + Some(LineageFailure::::code()), + ), + } +} + +/// The reference one named input resolves to through model and SQL function references. +#[pyfunction] +#[pyo3(signature = (*, model, input_name, models_by_name, functions_by_name))] +fn resolve_lineage_reference<'py>( + model: &Bound<'py, PyAny>, + input_name: &str, + models_by_name: Bound<'py, PyAny>, + functions_by_name: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = model.py(); + let graph = ProjectLineage { + models_by_name, + functions_by_name, + }; + let name: String = text(&model.getattr("name")?)?; + lineage_reference(&graph, (&name, references(model)?), input_name) + .map_err(|failure| lineage_error(py, failure)) +} + +/// Reject capped producers feeding a microbatch model's watermark inputs. +#[pyfunction(name = "validate_capped_watermark_inputs")] +#[pyo3(signature = (*, model, models_by_name, functions_by_name = None))] +fn reject_capped_watermark_inputs<'py>( + model: &Bound<'py, PyAny>, + models_by_name: Bound<'py, PyAny>, + functions_by_name: Option>, +) -> PyResult<()> { + let py: Python<'py> = model.py(); + if !microbatch_flag(model)? { + return Ok(()); + } + let functions_by_name: Bound<'py, PyAny> = match functions_by_name { + Some(functions) if functions.is_truthy()? => functions, + _ => PyDict::new(py).into_any(), + }; + let graph = ProjectLineage { + models_by_name, + functions_by_name, + }; + let name: String = text(&model.getattr("name")?)?; + let inputs: Vec = watermark_names(model)?; + match capped_watermark_input(&graph, (&name, references(model)?), inputs) { + Ok(None) => Ok(()), + Ok(Some(capped)) => Err(input_error( + py, + (COMPILE_EXCEPTIONS_MODULE, "CompileInputError"), + &capped.message(), + None, + )), + Err(failure) => Err(lineage_error(py, failure)), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_lineage_reference, module)?)?; + module.add_function(wrap_pyfunction!(reject_capped_watermark_inputs, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs new file mode 100644 index 0000000000..916e4d3de5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs @@ -0,0 +1,324 @@ +//! Microbatch counts, capped ranges and batch-limit configuration. + +use std::marker::PhantomData; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyString, PyTuple, PyTupleMethods}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_strategy::main::cap_integer_bounds::cap_integer_bounds; +use sqlbuild_analysis::planner_strategy::main::cap_timeline_bounds::cap_timeline_bounds; +use sqlbuild_analysis::planner_strategy::main::count_integer_batches::count_integer_batches; +use sqlbuild_analysis::planner_strategy::main::count_timeline_batches::count_timeline_batches; +use sqlbuild_analysis::planner_strategy::main::microbatch_limit_warning::microbatch_limit_warning as limit_warning; +use sqlbuild_analysis::planner_strategy::models::{CapDirection, CappedBound}; +use sqlbuild_analysis::planner_strategy::types::Timeline; +use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; +use sqlbuild_cursor::algebra::models::{DurationDigits, DurationSpan}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::cursor::durations::shifted_moment; +use crate::bindings::_helpers::plan_output::display::aligned_timestamp_bounds; +use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; +use crate::bindings::_helpers::planner_strategy::values::text; + +const CURSOR_MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; +const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +const INTEGER_CURSOR: &str = "integer"; +const TIMESTAMP_CURSOR: &str = "timestamp"; +const MONTHS_PER_YEAR: i64 = 12; +const CAP_FROM_START: &str = "cap_from_start"; +const CAP_FROM_END: &str = "cap_from_end"; +/// The limit actions that cap the planned range. +pub(crate) const CAP_ACTIONS: [&str; 2] = [CAP_FROM_END, CAP_FROM_START]; + +/// A start and end bound. +type BoundPair<'py> = (Bound<'py, PyAny>, Bound<'py, PyAny>); +/// An optional configured value. +type MaybeValue<'py> = Option>; + +/// Python `datetime` values one parsed duration apart. +struct DurationTimeline<'py> { + span: DurationSpan, + marker: PhantomData>, +} + +impl<'py> Timeline for DurationTimeline<'py> { + type Moment = Bound<'py, PyAny>; + type Error = PyErr; + + fn advance(&self, moment: &Bound<'py, PyAny>) -> PyResult> { + shifted_moment(moment, &self.span, 1) + } + + fn precedes(&self, left: &Bound<'py, PyAny>, right: &Bound<'py, PyAny>) -> PyResult { + left.lt(right) + } +} + +/// `Duration.parse(batch_size)` as a span, or `None` when it is not a duration. +fn duration_span(batch_size: &str) -> Option { + let digits: DurationDigits = parse_duration(batch_size)?; + let number = |text: &str| -> Option { + text.bytes().try_fold(0_i64, |value, digit| { + value + .checked_mul(10)? + .checked_add(i64::from(digit.checked_sub(b'0')?)) + }) + }; + Some(DurationSpan { + months: number(&digits.years)? * MONTHS_PER_YEAR + number(&digits.months)?, + days: number(&digits.days)?, + hours: number(&digits.hours)?, + minutes: number(&digits.minutes)?, + seconds: number(&digits.seconds)?, + }) +} + +/// The bounds of a `CursorBounds` when each is an instance of one of the cursor value `classes`. +fn typed_bounds<'py>( + bounds: &Bound<'py, PyAny>, + classes: &[&str], +) -> PyResult>> { + let py: Python<'py> = bounds.py(); + let (start, end) = (bounds.getattr("start")?, bounds.getattr("end")?); + for value in [&start, &end] { + let mut typed: bool = false; + for class in classes { + typed = typed || value.is_instance(&imported(py, CURSOR_MODELS_MODULE, class)?)?; + } + if !typed { + return Ok(None); + } + } + Ok(Some((start, end))) +} + +/// The integer values of integer bounds. +fn integer_bounds(bounds: &Bound<'_, PyAny>) -> PyResult> { + let Some((start, end)) = typed_bounds(bounds, &["IntegerValue"])? else { + return Ok(None); + }; + Ok(Some(( + start.getattr("value")?.extract()?, + end.getattr("value")?.extract()?, + ))) +} + +/// The aligned `datetime` bounds of date or timestamp bounds and the batch timeline. +fn timeline_bounds<'py>( + bounds: &Bound<'py, PyAny>, + batch_size: &str, +) -> PyResult, BoundPair<'py>)>> { + let Some(span) = duration_span(batch_size) else { + return Ok(None); + }; + let Some((start, end)) = typed_bounds(bounds, &["DateValue", "TimestampValue"])? else { + return Ok(None); + }; + let aligned: Bound<'py, PyTuple> = aligned_timestamp_bounds(&start, &end)?; + Ok(Some(( + DurationTimeline { + span, + marker: PhantomData, + }, + (aligned.get_item(0)?, aligned.get_item(1)?), + ))) +} + +/// Batches in one concrete cursor range. +#[pyfunction] +#[pyo3(signature = (*, bounds, batch_size, cursor_type, equal_bounds_are_batch = false))] +fn count_microbatches( + bounds: &Bound<'_, PyAny>, + batch_size: &str, + cursor_type: &str, + equal_bounds_are_batch: bool, +) -> PyResult { + if cursor_type == INTEGER_CURSOR { + return Ok(integer_bounds(bounds)?.map_or(0, |range| { + count_integer_batches(range, batch_size, equal_bounds_are_batch) + })); + } + if cursor_type != TIMESTAMP_CURSOR { + return Ok(0); + } + let Some((timeline, range)) = timeline_bounds(bounds, batch_size)? else { + return Ok(0); + }; + count_timeline_batches(&timeline, range, equal_bounds_are_batch) +} + +/// `CursorBounds(start=..., end=...)`. +fn cursor_bounds<'py>( + start: Bound<'py, PyAny>, + end: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = start.py(); + call_with( + &imported(py, MODELS_MODULE, "CursorBounds")?, + &[("start", start), ("end", end)], + ) +} + +/// `(value=value)` of the cursor values module. +fn cursor_value<'py>( + py: Python<'py>, + class: &str, + value: Bound<'py, PyAny>, +) -> PyResult> { + call_with( + &imported(py, CURSOR_MODELS_MODULE, class)?, + &[("value", value)], + ) +} + +/// One concrete range capped to its earliest or latest batch windows. +#[pyfunction] +#[pyo3(signature = (**options))] +fn cap_microbatch_bounds<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "cap_microbatch_bounds", + options, + ( + &[ + "bounds", + "batch_size", + "cursor_type", + "max_batches", + "action", + ], + &[], + ), + )?; + let bounds: Bound<'py, PyAny> = arguments.value("bounds")?; + let batch_size: String = text(&arguments.value("batch_size")?)?; + let cursor_type: String = text(&arguments.value("cursor_type")?)?; + let max_batches: i128 = arguments.value("max_batches")?.extract()?; + let direction: CapDirection = match text(&arguments.value("action")?)?.as_str() { + CAP_FROM_START => CapDirection::FromStart, + CAP_FROM_END => CapDirection::FromEnd, + _ => return Ok(bounds), + }; + let (start, end) = (bounds.getattr("start")?, bounds.getattr("end")?); + let capped: CappedBound> = if cursor_type == INTEGER_CURSOR { + let Some(range) = integer_bounds(&bounds)? else { + return Ok(bounds); + }; + let Some(capped) = cap_integer_bounds(range, &batch_size, max_batches, direction) else { + return Ok(bounds); + }; + match capped { + CappedBound::End(value) => CappedBound::End(cursor_value( + py, + "IntegerValue", + value.into_pyobject(py)?.into_any(), + )?), + CappedBound::Start(value) => CappedBound::Start(cursor_value( + py, + "IntegerValue", + value.into_pyobject(py)?.into_any(), + )?), + } + } else { + if cursor_type != TIMESTAMP_CURSOR { + return Ok(bounds); + } + let Some((timeline, range)) = timeline_bounds(&bounds, &batch_size)? else { + return Ok(bounds); + }; + match cap_timeline_bounds(&timeline, range, max_batches, direction)? { + CappedBound::End(value) => CappedBound::End(cursor_value(py, "TimestampValue", value)?), + CappedBound::Start(value) => { + CappedBound::Start(cursor_value(py, "TimestampValue", value)?) + } + } + }; + match capped { + CappedBound::End(capped_end) => cursor_bounds(start, capped_end), + CappedBound::Start(capped_start) => cursor_bounds(capped_start, end), + } +} + +/// The prominent warning or error message when one model exceeds its batch limit. +#[pyfunction] +#[pyo3(signature = (*, model_name, max_batches, batch_count, action))] +fn microbatch_limit_warning( + model_name: &str, + max_batches: Option, + batch_count: i128, + action: &Bound<'_, PyAny>, +) -> PyResult> { + let action: String = text(action)?; + let cap: bool = CAP_ACTIONS.contains(&action.as_str()); + Ok(limit_warning( + model_name, + (max_batches, batch_count), + (&action, cap), + )) +} + +/// A positive non-boolean integer, or `None`. +fn positive_integer<'py>(value: Option>) -> PyResult>> { + let Some(value) = value else { + return Ok(None); + }; + if value.is_instance_of::() && !value.is_instance_of::() && value.gt(0)? { + return Ok(Some(value)); + } + Ok(None) +} + +/// The configured batch limit and explicit action of validated model config values. +pub(crate) fn microbatch_limit_config<'py>( + values: &Bound<'py, PyAny>, +) -> PyResult<(MaybeValue<'py>, MaybeValue<'py>)> { + let py: Python<'py> = values.py(); + let raw: Bound<'py, PyAny> = values.call_method1("get", ("microbatch_limit",))?; + if let Ok(raw) = raw.downcast::() { + let max_batches: Option> = + positive_integer(raw.get_item("max_batches")?)?; + let action: Option> = raw + .get_item("action")? + .filter(|action| action.is_instance_of::()); + if let (Some(max_batches), Some(action)) = (max_batches, action) { + let action: Bound<'py, PyAny> = + imported(py, CONTRACT_TYPES_MODULE, "MicrobatchLimitAction")?.call1((action,))?; + return Ok((Some(max_batches), Some(action))); + } + } + let legacy: Bound<'py, PyAny> = values.call_method1("get", ("max_microbatches",))?; + Ok((positive_integer(Some(legacy))?, None)) +} + +/// `(max_batches, action)` from validated model config values. +#[pyfunction] +#[pyo3(signature = (*, values))] +fn resolve_microbatch_limit_config<'py>( + values: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = values.py(); + let (max_batches, action) = microbatch_limit_config(values)?; + PyTuple::new( + py, + [ + max_batches.unwrap_or_else(|| py.None().into_bound(py)), + action.unwrap_or_else(|| py.None().into_bound(py)), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(count_microbatches, module)?)?; + module.add_function(wrap_pyfunction!(cap_microbatch_bounds, module)?)?; + module.add_function(wrap_pyfunction!(microbatch_limit_warning, module)?)?; + module.add_function(wrap_pyfunction!(resolve_microbatch_limit_config, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs new file mode 100644 index 0000000000..0ec94542c6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod audits; +pub(crate) mod cursor_types; +pub(crate) mod lineage; +pub(crate) mod microbatches; +pub(crate) mod registration; +pub(crate) mod strategy; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs new file mode 100644 index 0000000000..e4a631cc0f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -0,0 +1,15 @@ +//! Register the planner strategy. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::planner_strategy::{ + audits, cursor_types, lineage, microbatches, strategy, +}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + audits::register(module)?; + cursor_types::register(module)?; + lineage::register(module)?; + microbatches::register(module)?; + strategy::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs new file mode 100644 index 0000000000..26418b9f29 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs @@ -0,0 +1,185 @@ +//! Per-model plan actions, schema actions and plan warnings. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_strategy::main::materialization_type::materialization_type; +use sqlbuild_analysis::planner_strategy::main::model_plan_action::model_plan_action; +use sqlbuild_analysis::planner_strategy::main::model_warnings::model_warnings; +use sqlbuild_analysis::planner_strategy::main::schema_actions::schema_actions; +use sqlbuild_analysis::planner_strategy::models::{ + ModelStrategy, PlanAction, PlanReason, SchemaActionKind, WarningInputs, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_changes::results::{MODELS_MODULE, planner_enum}; +use crate::bindings::_helpers::planner_strategy::values::{ + PLANNER_EXCEPTIONS_MODULE, change_facts, config_value, finding_rows, input_error, + materialization_value, model_materialization, optional_string, plan_warning, text, +}; + +fn action_value(action: PlanAction) -> &'static str { + match action { + PlanAction::CreateView => "create_view", + PlanAction::CreateTable => "create_table", + PlanAction::IncrementalAppend => "incremental_append", + PlanAction::IncrementalDeleteInsert => "incremental_delete_insert", + PlanAction::IncrementalMerge => "incremental_merge", + PlanAction::Snapshot => "snapshot", + PlanAction::Skip => "skip", + PlanAction::Custom => "custom", + } +} + +fn reason_value(reason: PlanReason) -> &'static str { + match reason { + PlanReason::FirstRun => "first_run", + PlanReason::Renamed => "renamed", + PlanReason::FullRefresh => "full_refresh", + PlanReason::QueryChanged => "query_changed", + PlanReason::FunctionChanged => "function_changed", + PlanReason::ConfigChanged => "config_changed", + PlanReason::SchemaChanged => "schema_changed", + PlanReason::NormalIncremental => "normal_incremental", + PlanReason::NoChange => "no_change", + PlanReason::Disabled => "disabled", + } +} + +/// The optional `on_schema_change` policy as text. +fn schema_policy(on_schema_change: &Bound<'_, PyAny>) -> PyResult> { + if on_schema_change.is_none() { + return Ok(None); + } + text(on_schema_change).map(Some) +} + +/// The `MaterializationType` of a model's `materialized` config: table when unset, custom when unknown. +#[pyfunction(name = "get_materialization_type")] +#[pyo3(signature = (model))] +fn materialization_of<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + planner_enum( + model.py(), + "MaterializationType", + materialization_value(model_materialization(model)?), + ) +} + +/// The plan action and reason for one model. +#[pyfunction] +#[pyo3(signature = (*, model, change_result, full_refresh))] +fn resolve_model_plan_action<'py>( + model: &Bound<'py, PyAny>, + change_result: &Bound<'py, PyAny>, + full_refresh: bool, +) -> PyResult> { + let py: Python<'py> = model.py(); + let enabled: Bound<'py, PyAny> = config_value(model, "enabled")?; + let strategy = ModelStrategy { + name: text(&model.getattr("name")?)?, + disabled: enabled.is_instance_of::() && !enabled.is_truthy()?, + materialization: model_materialization(model)?, + incremental_strategy: optional_string(&config_value(model, "incremental_strategy")?)?, + }; + let (action, reason) = + model_plan_action(&strategy, &change_facts(change_result)?, full_refresh).map_err( + |error| { + input_error( + py, + (PLANNER_EXCEPTIONS_MODULE, "PlannerInputError"), + &error.message(), + Some(error.code()), + ) + }, + )?; + PyTuple::new( + py, + [ + planner_enum(py, "PlanAction", action_value(action))?, + planner_enum(py, "PlanReason", reason_value(reason))?, + ], + ) +} + +/// Concrete schema change actions from findings and `on_schema_change`. +#[pyfunction] +#[pyo3(signature = (*, schema_findings, on_schema_change))] +fn resolve_schema_actions<'py>( + schema_findings: &Bound<'py, PyAny>, + on_schema_change: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = schema_findings.py(); + let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "SchemaAction")?; + let policy: Option = schema_policy(on_schema_change)?; + let mut actions: Vec> = Vec::new(); + for row in schema_actions(&finding_rows(schema_findings)?, policy.as_deref()) { + let kind: &str = match row.kind { + SchemaActionKind::AddColumn => "add_column", + SchemaActionKind::DropColumn => "drop_column", + SchemaActionKind::AlterColumnType => "alter_column_type", + }; + actions.push(call_with( + &class, + &[ + ("kind", planner_enum(py, "SchemaActionKind", kind)?), + ( + "column_name", + PyString::new(py, &row.column_name).into_any(), + ), + ("column_type", row.column_type.into_pyobject(py)?.into_any()), + ], + )?); + } + PyTuple::new(py, actions) +} + +/// Warnings for one model from its change detection and plan decisions. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_model_warnings<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_model_warnings", + options, + ( + &[ + "model_name", + "materialization_type", + "change_result", + "schema_actions", + "on_schema_change", + "type_enforcement", + ], + &[], + ), + )?; + let model_name: Bound<'py, PyAny> = arguments.value("model_name")?; + let inputs = WarningInputs { + model_name: text(&model_name)?, + materialization: materialization_type(Some(&text( + &arguments.value("materialization_type")?, + )?)), + has_schema_actions: arguments.flag("schema_actions")?, + on_schema_change: schema_policy(&arguments.value("on_schema_change")?)?, + type_enforcement: arguments.flag("type_enforcement")?, + }; + let rows = model_warnings(&inputs, &change_facts(&arguments.value("change_result")?)?); + let warnings: Vec> = rows + .into_iter() + .map(|row| plan_warning(&model_name, row)) + .collect::>()?; + PyTuple::new(py, warnings) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(materialization_of, module)?)?; + module.add_function(wrap_pyfunction!(resolve_model_plan_action, module)?)?; + module.add_function(wrap_pyfunction!(resolve_schema_actions, module)?)?; + module.add_function(wrap_pyfunction!(build_model_warnings, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs new file mode 100644 index 0000000000..5101af6fd6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs @@ -0,0 +1,168 @@ +//! Conversions between the planner's Python models and the native strategy rows. + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; + +use sqlbuild_analysis::planner_changes::models::{ + SchemaChangeKind, SchemaColumnSource, SchemaFindingRow, +}; +use sqlbuild_analysis::planner_strategy::main::materialization_type::materialization_type; +use sqlbuild_analysis::planner_strategy::models::{ + ChangeFacts, MaterializationType, PlanChangeKind, WarningRow, WarningSeverity, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_changes::results::{MODELS_MODULE, planner_enum}; + +pub(crate) const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +pub(crate) const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const YML_SOURCE: &str = "yml"; +const FULL_BACKFILL: &str = "full"; +const FORWARD_ONLY_BACKFILL: &str = "forward"; + +/// `str(value)` of a Python string or string enum. +pub(crate) fn text(value: &Bound<'_, PyAny>) -> PyResult { + value.extract() +} + +/// A string value, or `None` for `None` and anything that is not a string. +pub(crate) fn optional_string(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_instance_of::() { + return text(value).map(Some); + } + Ok(None) +} + +/// `values.get(key)` of a model's config. +pub(crate) fn config_value<'py>( + model: &Bound<'py, PyAny>, + key: &str, +) -> PyResult> { + model + .getattr("config")? + .getattr("values")? + .call_method1("get", (key,)) +} + +/// The materialization type of a model's `materialized` config. +pub(crate) fn model_materialization(model: &Bound<'_, PyAny>) -> PyResult { + let materialized: Option = optional_string(&config_value(model, "materialized")?)?; + Ok(materialization_type(materialized.as_deref())) +} + +/// The `MaterializationType` value of a materialization. +pub(crate) fn materialization_value(materialization: MaterializationType) -> &'static str { + match materialization { + MaterializationType::View => "view", + MaterializationType::Table => "table", + MaterializationType::Incremental => "incremental", + MaterializationType::Snapshot => "snapshot", + MaterializationType::Seed => "seed", + MaterializationType::Custom => "custom", + } +} + +fn change_kind(value: &str) -> PlanChangeKind { + match value { + "first_run" => PlanChangeKind::FirstRun, + "renamed" => PlanChangeKind::Renamed, + "query_changed" => PlanChangeKind::QueryChanged, + "function_changed" => PlanChangeKind::FunctionChanged, + "config_changed" => PlanChangeKind::ConfigChanged, + "schema_changed" => PlanChangeKind::SchemaChanged, + _ => PlanChangeKind::NoChange, + } +} + +/// One `SchemaFinding` as a native row. +fn finding_row(finding: &Bound<'_, PyAny>) -> PyResult { + let kind: SchemaChangeKind = match text(&finding.getattr("kind")?)?.as_str() { + "column_added" => SchemaChangeKind::ColumnAdded, + "column_removed" => SchemaChangeKind::ColumnRemoved, + _ => SchemaChangeKind::ColumnTypeChanged, + }; + let source: SchemaColumnSource = if text(&finding.getattr("source")?)? == YML_SOURCE { + SchemaColumnSource::Yml + } else { + SchemaColumnSource::SqlAnalysis + }; + Ok(SchemaFindingRow { + kind, + column_name: text(&finding.getattr("column_name")?)?, + source, + expected_type: finding.getattr("expected_type")?.extract()?, + actual_type: finding.getattr("actual_type")?.extract()?, + }) +} + +/// A tuple of `SchemaFinding` values as native rows. +pub(crate) fn finding_rows(findings: &Bound<'_, PyAny>) -> PyResult> { + findings + .try_iter()? + .map(|finding| finding_row(&finding?)) + .collect() +} + +/// The parts of a `ChangeDetectionResult` the strategy reads. +pub(crate) fn change_facts(change: &Bound<'_, PyAny>) -> PyResult { + let backfill: String = text(&change.getattr("backfill")?.getattr("action")?)?; + Ok(ChangeFacts { + kind: change_kind(&text(&change.getattr("change_kind")?)?), + backfill_full: backfill == FULL_BACKFILL, + backfill_forward_only: backfill == FORWARD_ONLY_BACKFILL, + query_changed: change.getattr("query_changed")?.is_truthy()?, + changed_functions: change + .getattr("changed_functions")? + .try_iter()? + .map(|name| text(&name?)) + .collect::>()?, + config_changed: change.getattr("config_changed")?.is_truthy()?, + schema_findings: finding_rows(&change.getattr("schema_findings")?)?, + recorded_build_relation_missing: change + .getattr("recorded_build_relation_missing")? + .is_truthy()?, + }) +} + +/// `PlanWarning(model_name=..., severity=..., message=..., code=...)`. +pub(crate) fn plan_warning<'py>( + model_name: &Bound<'py, PyAny>, + row: WarningRow, +) -> PyResult> { + let py: Python<'py> = model_name.py(); + let severity: &str = match row.severity { + WarningSeverity::Error => "error", + WarningSeverity::Warning => "warning", + WarningSeverity::Info => "info", + }; + call_with( + &imported(py, MODELS_MODULE, "PlanWarning")?, + &[ + ("model_name", model_name.clone()), + ("severity", planner_enum(py, "WarningSeverity", severity)?), + ("message", PyString::new(py, &row.message).into_any()), + ("code", row.code.into_pyobject(py)?.into_any()), + ], + ) +} + +/// `.(message, code=code)` raised as an error. +pub(crate) fn input_error( + py: Python<'_>, + (module, class): (&str, &str), + message: &str, + code: Option<&str>, +) -> PyErr { + let raised: PyResult> = imported(py, module, class).and_then(|class| { + let options: Bound<'_, PyDict> = PyDict::new(py); + if let Some(code) = code { + options.set_item("code", code)?; + } + class.call((message,), Some(&options)) + }); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs index 5f1224cc0d..b58d1f0cc0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod declared_relations; +pub(crate) mod native_resources; pub(crate) mod registration; pub(crate) mod resources; pub(crate) mod syntax_checks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs new file mode 100644 index 0000000000..87c54458bf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs @@ -0,0 +1,480 @@ +//! The native project assembly request built from compile inputs, and its resource facts. + +use pyo3::exceptions::{PyException, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyBoolMethods, PyInt, PyList, PyString, PyStringMethods, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; +use sqlbuild_analysis::assembly::project::errors::AssemblyError; +use sqlbuild_analysis::assembly::project::main::assemble_project_resources::assemble_project_resources; +use sqlbuild_analysis::assembly::project::main::environment_template_names::environment_template_names; +use sqlbuild_analysis::assembly::project::models::{ + AuditFacts, FunctionFacts, InputRead, ModelFacts, Namespace, OpaqueRender, OpaqueVariable, + ProjectRequest, ProjectResources, Reference, ResourceFailure, ResourceSlot, SeedDefaults, + SeedFacts, SourceFacts, SyntaxCheck, TargetNamespace, Variable, +}; +use sqlbuild_core::panics::main::native_failure::native_failure; +use sqlbuild_model_config::templates::models::Scalar; + +use crate::bindings::_helpers::authored_values::project_vars::project_var_text; +use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_error}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::model_config::template_data::record_reads; +use crate::bindings::types::CompilerDetach; + +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const ASSEMBLY_MODELS_MODULE: &str = "sqlbuild.compiler.project_assembly.models"; +const OS_MODULE: &str = "os"; +const PROJECT_ASSEMBLY_CONTEXT: &str = "native project assembly"; +const GENERIC_DIALECT: &str = "generic"; +const RENDER_LABEL: &str = "\u{0}"; +const SOURCE_SLOT: &str = "source"; +const SEED_SLOT: &str = "seed"; +const AUDIT_SLOT: &str = "audit"; +const ENVIRONMENT_READ: &str = "env"; +const CONTEXT_READ: &str = "ctx"; +const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; + +/// `(kind, name, package)`. +type ReferenceRow = (String, String, Option); +/// `(name, managed, database, schema)`. +type SourceRow = (String, bool, Option, Option); +/// `(kind, name, tags, references)`. +type FunctionRow = (String, String, Vec, Vec); +/// `(references, attached (kind, name))`. +type AuditRow = (Vec, Option<(String, String)>); + +/// The request's named variables and the Python exceptions opaque ones raise, by index. +type RequestVariables<'py> = (Vec<(String, Variable)>, Vec>); + +/// How one opaque variable renders as text, or the Python exception rendering it raises. +enum Render<'py> { + Text(String), + Rejected(String), + Raises(Bound<'py, PyAny>), +} + +/// The plain scalar `value` crosses as: `None`, exact `bool`, `str` or 64-bit `int`. +fn plain_scalar(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(Some(Scalar::Null)); + } + if let Ok(flag) = value.downcast_exact::() { + return Ok(Some(Scalar::Bool(flag.is_true()))); + } + if let Ok(text) = value.downcast_exact::() { + return Ok(Some(Scalar::Text(text.to_str()?.to_owned()))); + } + if value.is_exact_instance_of::() { + return Ok(match value.extract::() { + Ok(number) => Some(Scalar::Text(number.to_string())), + Err(_) => None, + }); + } + Ok(None) +} + +/// `render_project_var_text` under a label no message starts with otherwise. +fn opaque_render<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + match project_var_text(value, RENDER_LABEL) { + Ok(text) => Ok(Render::Text(text)), + Err(error) if !error.is_instance_of::(py) => Err(error), + Err(error) => { + let raised: Bound<'py, PyAny> = error.value(py).clone().into_any(); + if error.is_instance_of::(py) { + let message: String = raised.str()?.extract()?; + if let Some(rest) = message.strip_prefix(RENDER_LABEL) { + return Ok(Render::Rejected(rest.to_owned())); + } + } + Ok(Render::Raises(raised)) + } + } +} + +/// Each effective var as the native request reads it, and the exceptions opaque ones raise. +fn variables<'py>(effective_vars: &Bound<'py, PyAny>) -> PyResult> { + let mut rows: Vec<(String, Variable)> = Vec::new(); + let mut held: Vec> = Vec::new(); + for item in effective_vars.call_method0("items")?.try_iter()? { + let (name, value): (String, Bound<'py, PyAny>) = item?.extract()?; + if let Some(scalar) = plain_scalar(&value)? { + rows.push((name, Variable::Scalar(scalar))); + continue; + } + let render: OpaqueRender = match opaque_render(&value)? { + Render::Text(text) => OpaqueRender::Text(text), + Render::Rejected(message) => OpaqueRender::Rejected(message), + Render::Raises(error) => { + held.push(error); + OpaqueRender::Raises(held.len() - 1) + } + }; + let text: String = value.str()?.extract::()?; + rows.push((name, Variable::Opaque(OpaqueVariable { text, render }))); + } + Ok((rows, held)) +} + +fn optional_text(value: &Bound<'_, PyAny>, attribute: &str) -> PyResult> { + value.getattr(attribute)?.extract() +} + +/// Python's own `os.environ` lookup of every name an `ENV:` template may read. +fn environment_rows( + py: Python<'_>, + values: &[Option], +) -> PyResult)>> { + let environment: Bound<'_, PyAny> = imported(py, OS_MODULE, "environ")?; + environment_template_names(values) + .into_iter() + .map(|name| { + let value: Option = environment.call_method1("get", (&name,))?.extract()?; + Ok((name, value)) + }) + .collect() +} + +fn references(rows: Vec) -> Vec { + rows.into_iter() + .map(|(kind, name, package)| Reference { + kind, + name, + package, + }) + .collect() +} + +/// Model config tags as the project graph indexes them. +fn tags(value: &Bound<'_, PyAny>) -> PyResult> { + if !(value.is_instance_of::() || value.is_instance_of::()) { + return Ok(Vec::new()); + } + value + .try_iter()? + .map(|item| item?.str()?.extract::()) + .collect() +} + +fn model_facts(model_input: &Bound<'_, PyAny>, checks: &Bound<'_, PyAny>) -> PyResult { + let model_file: Bound<'_, PyAny> = model_input.getattr("model_file")?; + let mut reference_rows: Vec = Vec::new(); + for reference in model_input.getattr("references")?.try_iter()? { + let reference: Bound<'_, PyAny> = reference?; + reference_rows.push(( + reference.getattr("ref_kind")?.str()?.extract::()?, + reference.getattr("ref_name")?.extract()?, + reference.getattr("ref_package")?.extract()?, + )); + } + let mut syntax_checks: Vec = Vec::new(); + for check in checks.try_iter()? { + let (sql, placeholders): (String, Bound<'_, PyAny>) = check?.extract()?; + let placeholders: Pairs = if placeholders.is_truthy()? { + placeholders + .call_method0("items")? + .try_iter()? + .map(|pair| pair?.extract()) + .collect::>()? + } else { + Vec::new() + }; + syntax_checks.push(SyntaxCheck { sql, placeholders }); + } + Ok(ModelFacts { + name: model_file + .getattr("file_path")? + .getattr("stem")? + .extract()?, + directory: model_file + .getattr("relative_path")? + .getattr("parent")? + .str()? + .to_string(), + tags: tags( + &model_input + .getattr("config")? + .getattr("values")? + .call_method1("get", ("tags",))?, + )?, + references: references(reference_rows), + syntax_checks, + }) +} + +/// Every item of `inputs.`. +fn items<'py>(inputs: &Bound<'py, PyAny>, attribute: &str) -> PyResult>> { + inputs.getattr(attribute)?.try_iter()?.collect() +} + +/// The native assembly request for compile inputs, and the exceptions opaque vars raise. +fn project_request<'py>( + inputs: &Bound<'py, PyAny>, + dialect: Option, + syntax_checks: &Bound<'py, PyAny>, +) -> PyResult<(ProjectRequest, Vec>)> { + let py: Python<'py> = inputs.py(); + let (variables, held) = variables(&inputs.getattr("effective_vars")?)?; + let target: Bound<'py, PyAny> = inputs.getattr("effective_target")?; + let target: Option = if target.is_none() { + None + } else { + Some(TargetNamespace { + database: optional_text(&target, "database")?, + schema: optional_text(&target, "schema")?, + loader_schema: optional_text(&target, "loader_schema")?, + }) + }; + let defaults: Bound<'py, PyAny> = inputs.getattr("project_config")?.getattr("defaults")?; + let defaults = SeedDefaults { + database: optional_text(&defaults, "database")?, + schema: optional_text(&defaults, "schema")?, + seed_database: optional_text(&defaults, "seed_database")?, + seed_schema: optional_text(&defaults, "seed_schema")?, + }; + let seed_inputs: Vec> = items(inputs, "seed_inputs")?; + let mut seeds: Vec = Vec::with_capacity(seed_inputs.len()); + for seed_input in &seed_inputs { + let entry: Bound<'py, PyAny> = seed_input.getattr("schema_entry")?; + seeds.push(SeedFacts { + name: entry.getattr("name")?.extract()?, + tags: entry + .getattr("tags")? + .try_iter()? + .map(|tag| tag?.extract()) + .collect::>()?, + database: optional_text(&entry, "database")?, + schema: optional_text(&entry, "schema")?, + }); + } + let mut template_values: Vec> = Vec::new(); + if let Some(target) = &target { + template_values.extend([ + target.database.clone(), + target.schema.clone(), + target.loader_schema.clone(), + ]); + } + template_values.extend([ + defaults.database.clone(), + defaults.schema.clone(), + defaults.seed_database.clone(), + defaults.seed_schema.clone(), + ]); + for seed in &seeds { + template_values.extend([seed.database.clone(), seed.schema.clone()]); + } + let model_inputs: Vec> = items(inputs, "model_inputs")?; + let checks: Vec> = syntax_checks.try_iter()?.collect::>()?; + if checks.len() != model_inputs.len() { + return Err(PyValueError::new_err( + "zip() arguments have different lengths: model inputs and syntax checks", + )); + } + let models: Vec = model_inputs + .iter() + .zip(&checks) + .map(|(model_input, checks)| model_facts(model_input, checks)) + .collect::>()?; + let rows = |attribute: &str| -> PyResult>> { + items(inputs, attribute)? + .into_iter() + .map(|input| input.getattr("assembly_row")) + .collect() + }; + let sources: Vec = rows("source_inputs")? + .into_iter() + .map(|row| { + let (name, managed, database, schema): SourceRow = row.extract()?; + Ok(SourceFacts { + name, + managed, + database, + schema, + }) + }) + .collect::>()?; + let functions: Vec = rows("sql_function_inputs")? + .into_iter() + .map(|row| { + let (kind, name, tags, reference_rows): FunctionRow = row.extract()?; + Ok(FunctionFacts { + kind, + name, + tags, + references: references(reference_rows), + }) + }) + .collect::>()?; + let audits: Vec = rows("audit_inputs")? + .into_iter() + .map(|row| { + let (reference_rows, attached): AuditRow = row.extract()?; + Ok(AuditFacts { + references: references(reference_rows), + attached, + }) + }) + .collect::>()?; + let request = ProjectRequest { + dialect: dialect + .filter(|dialect| !dialect.is_empty()) + .unwrap_or_else(|| GENERIC_DIALECT.to_owned()), + target, + defaults, + variables, + environment: environment_rows(py, &template_values)?, + models, + sources, + seeds, + functions, + audits, + }; + Ok((request, held)) +} + +/// The exception Python's assembly raises at the failing resource. +fn failure_error<'py>( + py: Python<'py>, + error: AssemblyError, + held: &[Bound<'py, PyAny>], +) -> PyResult> { + match error { + AssemblyError::CompileInput { message, help } => { + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?.call( + (message,), + Some(&keywords( + py, + &[("help", help.into_pyobject(py)?.into_any())], + )?), + ) + } + AssemblyError::Value(message) => { + Ok(PyValueError::new_err(message).value(py).clone().into_any()) + } + AssemblyError::Raise(index) => match held.get(index) { + Some(error) => Ok(error.clone()), + None => Ok(NativeCompilerError::new_err(format!( + "{NATIVE_FAILURE_PREFIX}{PROJECT_ASSEMBLY_CONTEXT}: unknown failure kind 'held'" + )) + .value(py) + .clone() + .into_any()), + }, + AssemblyError::Internal(reason) => Err(compiler_error(native_failure( + PROJECT_ASSEMBLY_CONTEXT, + &reason, + ))), + } +} + +fn failure_object<'py>( + py: Python<'py>, + failure: ResourceFailure, + held: &[Bound<'py, PyAny>], +) -> PyResult> { + let slot: &str = match failure.slot { + ResourceSlot::Source => SOURCE_SLOT, + ResourceSlot::Seed => SEED_SLOT, + ResourceSlot::Audit => AUDIT_SLOT, + }; + call_with( + &imported(py, ASSEMBLY_MODELS_MODULE, "NativeResourceFailure")?, + &[ + ("slot", PyString::new(py, slot).into_any()), + ("index", failure.index.into_pyobject(py)?.into_any()), + ("error", failure_error(py, failure.error, held)?), + ], + ) +} + +fn namespace_row( + namespace: Namespace, +) -> ( + Option, + Option, + Option, + Option, +) { + ( + namespace.database, + namespace.schema, + namespace.logical_database, + namespace.logical_schema, + ) +} + +/// `NativeProjectResources` for the native result, replaying its template reads first. +fn resources_object<'py>( + py: Python<'py>, + resources: ProjectResources, + held: &[Bound<'py, PyAny>], +) -> PyResult> { + record_reads( + py, + resources.reads.iter().map(|read| match read { + InputRead::Environment(name) => (ENVIRONMENT_READ, name.as_str()), + InputRead::Context(name) => (CONTEXT_READ, name.as_str()), + }), + )?; + let tuple = |values: Bound<'py, PyAny>| -> PyResult> { + Ok(PyTuple::new(py, values.try_iter()?.collect::>>()?)?.into_any()) + }; + let failure: Bound<'py, PyAny> = match resources.failure { + Some(failure) => failure_object(py, failure, held)?, + None => py.None().into_bound(py), + }; + let seeds: Vec<_> = resources.seeds.into_iter().map(namespace_row).collect(); + call_with( + &imported(py, ASSEMBLY_MODELS_MODULE, "NativeProjectResources")?, + &[ + ( + "model_deps", + tuple(resources.model_deps.into_pyobject(py)?.into_any())?, + ), + ( + "model_syntax_valid", + tuple(resources.model_syntax_valid.into_pyobject(py)?.into_any())?, + ), + ( + "source_namespaces", + tuple(resources.sources.into_pyobject(py)?.into_any())?, + ), + ( + "seed_namespaces", + tuple(seeds.into_pyobject(py)?.into_any())?, + ), + ( + "function_deps", + tuple(resources.function_deps.into_pyobject(py)?.into_any())?, + ), + ( + "audit_scope_deps", + tuple(resources.audit_deps.into_pyobject(py)?.into_any())?, + ), + ("failure", failure), + ], + ) +} + +/// Return the resource facts and the resource Python's assembly raises at, if any. +#[pyfunction] +#[pyo3(signature = (*, inputs, dialect, syntax_checks))] +fn assemble_native_project_resources<'py>( + inputs: &Bound<'py, PyAny>, + dialect: Option, + syntax_checks: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let (request, held) = project_request(inputs, dialect, syntax_checks)?; + let resources: ProjectResources = py + .compiler_detach(|| assemble_project_resources(&request)) + .map_err(|reason| compiler_error(native_failure(PROJECT_ASSEMBLY_CONTEXT, &reason)))?; + resources_object(py, resources, &held) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(assemble_native_project_resources, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs index efd98b398b..9823ce1309 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs @@ -3,11 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::project_assembly::{ - declared_relations, resources, syntax_checks, syntax_validation, + declared_relations, native_resources, resources, syntax_checks, syntax_validation, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { declared_relations::register(module)?; + native_resources::register(module)?; syntax_validation::register(module)?; syntax_checks::register(module)?; resources::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs index a201a7615d..77eb7ebbfa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs @@ -7,123 +7,16 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyString, PyStringMethods}; use pyo3::{PyErr, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::analysis_session::types::Pairs; -use sqlbuild_analysis::assembly::project::errors::AssemblyError; -use sqlbuild_analysis::assembly::project::main::assemble_project_resources::assemble_project_resources; use sqlbuild_analysis::assembly::project::main::check_sql_syntax::check_sql_syntax; -use sqlbuild_analysis::assembly::project::main::environment_template_names::environment_template_names as template_names; use sqlbuild_analysis::assembly::project::main::sql_syntax_error::sql_syntax_error as check_syntax_error; -use sqlbuild_analysis::assembly::project::models::{ - AuditFacts, FunctionFacts, InputRead, ModelFacts, Namespace, OpaqueRender, OpaqueVariable, - ProjectRequest, ProjectResources, Reference, ResourceFailure, ResourceSlot, SeedDefaults, - SeedFacts, SourceFacts, SyntaxCheck, SyntaxFailure, SyntaxMode, TargetNamespace, Variable, -}; -use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxFailure, SyntaxMode}; use sqlbuild_core::panics::main::native_failure::native_failure; -use sqlbuild_model_config::templates::models::Scalar; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::types::CompilerDetach; -const PROJECT_ASSEMBLY_CONTEXT: &str = "native project assembly"; const SYNTAX_CHECK_CONTEXT: &str = "native SQL syntax check"; -const TEXT_VARIABLE: &str = "text"; -const BOOLEAN_VARIABLE: &str = "bool"; -const NONE_VARIABLE: &str = "none"; -const OPAQUE_VARIABLE: &str = "opaque"; -const RENDERED_TEXT: &str = "text"; -const RENDER_REJECTED: &str = "rejected"; -const RENDER_RAISES: &str = "raises"; -const SOURCE_SLOT: &str = "source"; -const SEED_SLOT: &str = "seed"; -const AUDIT_SLOT: &str = "audit"; -const COMPILE_INPUT_ERROR: &str = "compile_input"; -const VALUE_ERROR: &str = "value"; -const HELD_ERROR: &str = "held"; -const TRUE_TEXT: &str = "1"; -const ENVIRONMENT_READ: &str = "env"; -const CONTEXT_READ: &str = "ctx"; - -/// `(kind, name, package)`. -type ReferenceRow = (String, String, Option); -/// `(database, schema, loader schema)`. -type TargetRow = (Option, Option, Option); -/// `(database, schema, seed database, seed schema)`. -type DefaultsRow = ( - Option, - Option, - Option, - Option, -); -/// `(name, kind, text, render)`: kind `text`, `bool`, `none` or `opaque` with a render pair. -type VariableRow = (String, String, String, Option<(String, String)>); -/// `(resource list, index, error class, message, help, held exception index)`. -type FailureRow = ( - &'static str, - usize, - &'static str, - String, - Option, - Option, -); -/// `(name, directory, tags, references, [(sql, placeholders)])`. -type ModelRow = ( - String, - String, - Vec, - Vec, - Vec<(String, Pairs)>, -); -/// `(name, managed, database, schema)`. -type SourceRow = (String, bool, Option, Option); -/// `(name, tags, database, schema)`. -type SeedRow = (String, Vec, Option, Option); -/// `(kind, name, tags, references)`. -type FunctionRow = (String, String, Vec, Vec); -/// `(references, attached (kind, name))`. -type AuditRow = (Vec, Option<(String, String)>); -/// The dialect, target, defaults, variables, environment and resources one project assembles. -type RequestRow = ( - String, - Option, - DefaultsRow, - Vec, - Vec<(String, Option)>, - Vec, - Vec, - Vec, - Vec, - Vec, -); -/// `(database, schema, logical database, logical schema)`. -type NamespaceRow = ( - Option, - Option, - Option, - Option, -); -/// Deps, managed source and seed namespaces, and the `(env or ctx, name)` template reads. -type ResourcesRow = ( - Vec>, - Vec, Option)>>, - Vec, - Vec>, - Vec>, - Vec<(&'static str, String)>, - Vec, - Option, -); - -/// The project's resource facts up to the resource Python's assembly raises at. -#[pyfunction] -fn assemble_project_resource_facts(py: Python<'_>, request: RequestRow) -> PyResult { - let request: ProjectRequest = project_request(request)?; - let resources: ProjectResources = py - .compiler_detach(|| assemble_project_resources(&request)) - .map_err(|reason| compiler_error(native_failure(PROJECT_ASSEMBLY_CONTEXT, &reason)))?; - resources_row(resources) -} - /// Python's syntax error message for one SQL string, None where it parses. #[pyfunction] #[pyo3(signature = (sql, placeholders, dialect, parse_one=false))] @@ -196,211 +89,8 @@ pub(crate) fn syntax_valid( .map_err(syntax_value_error) } -fn project_request(request: RequestRow) -> PyResult { - let ( - dialect, - target, - defaults, - variables, - environment, - models, - sources, - seeds, - functions, - audits, - ) = request; - let (database, schema, seed_database, seed_schema) = defaults; - Ok(ProjectRequest { - dialect, - target: target.map(|(database, schema, loader_schema)| TargetNamespace { - database, - schema, - loader_schema, - }), - defaults: SeedDefaults { - database, - schema, - seed_database, - seed_schema, - }, - variables: variables - .into_iter() - .map(variable) - .collect::>()?, - environment, - models: models.into_iter().map(model).collect(), - sources: sources - .into_iter() - .map(|(name, managed, database, schema)| SourceFacts { - name, - managed, - database, - schema, - }) - .collect(), - seeds: seeds - .into_iter() - .map(|(name, tags, database, schema)| SeedFacts { - name, - tags, - database, - schema, - }) - .collect(), - functions: functions - .into_iter() - .map(|(kind, name, tags, rows)| FunctionFacts { - kind, - name, - tags, - references: references(rows), - }) - .collect(), - audits: audits - .into_iter() - .map(|(rows, attached)| AuditFacts { - references: references(rows), - attached, - }) - .collect(), - }) -} - -fn variable(row: VariableRow) -> PyResult<(String, Variable)> { - let (name, kind, text, render) = row; - let value: Variable = match (kind.as_str(), render) { - (TEXT_VARIABLE, _) => Variable::Scalar(Scalar::Text(text)), - (BOOLEAN_VARIABLE, _) => Variable::Scalar(Scalar::Bool(text == TRUE_TEXT)), - (NONE_VARIABLE, _) => Variable::Scalar(Scalar::Null), - (OPAQUE_VARIABLE, Some((render_kind, payload))) => Variable::Opaque(OpaqueVariable { - text, - render: opaque_render(&render_kind, payload)?, - }), - _ => { - return Err(compiler_error(native_failure( - PROJECT_ASSEMBLY_CONTEXT, - &format!("an unknown variable kind '{kind}'"), - ))); - } - }; - Ok((name, value)) -} - -fn opaque_render(kind: &str, payload: String) -> PyResult { - match kind { - RENDERED_TEXT => Ok(OpaqueRender::Text(payload)), - RENDER_REJECTED => Ok(OpaqueRender::Rejected(payload)), - RENDER_RAISES => payload.parse().map(OpaqueRender::Raises).map_err(|_| { - compiler_error(native_failure( - PROJECT_ASSEMBLY_CONTEXT, - "an opaque variable's held exception index is not a number", - )) - }), - _ => Err(compiler_error(native_failure( - PROJECT_ASSEMBLY_CONTEXT, - &format!("an unknown variable rendering '{kind}'"), - ))), - } -} - -fn model(row: ModelRow) -> ModelFacts { - let (name, directory, tags, rows, checks) = row; - ModelFacts { - name, - directory, - tags, - references: references(rows), - syntax_checks: checks - .into_iter() - .map(|(sql, placeholders)| SyntaxCheck { sql, placeholders }) - .collect(), - } -} - -fn references(rows: Vec) -> Vec { - rows.into_iter() - .map(|(kind, name, package)| Reference { - kind, - name, - package, - }) - .collect() -} - -fn resources_row(resources: ProjectResources) -> PyResult { - let failure: Option = resources.failure.map(failure_row).transpose()?; - Ok(( - resources.model_deps, - resources.sources, - resources.seeds.into_iter().map(namespace_row).collect(), - resources.function_deps, - resources.audit_deps, - resources.reads.into_iter().map(read_row).collect(), - resources.model_syntax_valid, - failure, - )) -} - -/// `(resource list, index, error class, message, help, held exception index)`. -fn failure_row(failure: ResourceFailure) -> PyResult { - let slot: &'static str = match failure.slot { - ResourceSlot::Source => SOURCE_SLOT, - ResourceSlot::Seed => SEED_SLOT, - ResourceSlot::Audit => AUDIT_SLOT, - }; - Ok(match failure.error { - AssemblyError::CompileInput { message, help } => ( - slot, - failure.index, - COMPILE_INPUT_ERROR, - message, - help, - None, - ), - AssemblyError::Value(message) => (slot, failure.index, VALUE_ERROR, message, None, None), - AssemblyError::Raise(held) => ( - slot, - failure.index, - HELD_ERROR, - String::new(), - None, - Some(held), - ), - AssemblyError::Internal(reason) => { - return Err(compiler_error(native_failure( - PROJECT_ASSEMBLY_CONTEXT, - &reason, - ))); - } - }) -} - -fn read_row(read: InputRead) -> (&'static str, String) { - match read { - InputRead::Environment(name) => (ENVIRONMENT_READ, name), - InputRead::Context(name) => (CONTEXT_READ, name), - } -} - -fn namespace_row(namespace: Namespace) -> NamespaceRow { - ( - namespace.database, - namespace.schema, - namespace.logical_database, - namespace.logical_schema, - ) -} - -/// The distinct `ENV:` names target templates read, in first-seen order. -#[pyfunction] -fn environment_template_names(values: Vec>) -> Vec { - template_names(&values) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(assemble_project_resource_facts, module)?)?; module.add_function(wrap_pyfunction!(check_native_sql_syntax, module)?)?; module.add_function(wrap_pyfunction!(sql_syntax_error, module)?)?; - module.add_function(wrap_pyfunction!(environment_template_names, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs index 80fd22543b..3ba20fef2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs @@ -5,11 +5,9 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::project_assembly::syntax_validation::hook_sql_statements; -const CURSOR_INTRINSICS_MODULE: &str = - "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; fn config_value<'py>(model_input: &Bound<'py, PyAny>, key: &str) -> PyResult> { @@ -46,17 +44,15 @@ pub(crate) fn model_placeholders<'py>( /// The model query with cursor intrinsics replaced by typed literals for static analysis. fn analysis_sql<'py>(model_input: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = model_input.py(); - call_with( - &imported( + Ok(PyString::new( + py, + &analysis_cursor_sql( py, - CURSOR_INTRINSICS_MODULE, - "cursor_intrinsics_analysis_sql", + &model_input.getattr("query_sql")?.extract::()?, + &config_value(model_input, "cursor_type")?, )?, - &[ - ("sql", model_input.getattr("query_sql")?), - ("cursor_type", config_value(model_input, "cursor_type")?), - ], ) + .into_any()) } fn syntax_checks<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs index 3e27210052..b6b7f7fe44 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs @@ -7,6 +7,7 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::project::main::sql_analysis_opt_out_help::sql_analysis_opt_out_help; use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; use crate::bindings::_helpers::project_assembly::resources::syntax_error_message; @@ -16,8 +17,6 @@ const SQL_HOOK_ENTRY: &str = "SqlHookEntry"; const PYTHON_HOOK_ENTRY: &str = "PythonHookEntry"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_INPUT_ERROR: &str = "CompileInputError"; -const VALIDATION_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.validation"; -const OPT_OUT_HELP: &str = "sql_analysis_opt_out_help"; const GENERIC_DIALECT: &str = "generic"; /// `(placeholder defaults, dialect)` a statement parses with. @@ -43,14 +42,8 @@ struct HookSite<'a, 'py> { /// `CompileInputError(message, help=sql_analysis_opt_out_help(model_owned=True))`. fn syntax_error(py: Python<'_>, message: String) -> PyErr { let raised = || -> PyResult { - let help_arguments: Bound<'_, PyDict> = PyDict::new(py); - help_arguments.set_item("model_owned", true)?; - let help: Bound<'_, PyAny> = py - .import(VALIDATION_MODULE)? - .getattr(OPT_OUT_HELP)? - .call((), Some(&help_arguments))?; let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("help", help)?; + keywords.set_item("help", sql_analysis_opt_out_help(true))?; let error: Bound<'_, PyAny> = py .import(COMPILE_ERRORS_MODULE)? .getattr(COMPILE_INPUT_ERROR)? @@ -248,7 +241,15 @@ pub(crate) fn hook_sql_statements(value: Bound<'_, PyAny>) -> PyResult String { + sql_analysis_opt_out_help(model_owned) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(opt_out_help, module)?)?; module.add_function(wrap_pyfunction!(validate_hook_sql_syntax, module)?)?; module.add_function(wrap_pyfunction!(validate_sql_syntax, module)?)?; module.add_function(wrap_pyfunction!(hook_sql_statements, module)?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs index 5341d58eed..01c92d2a40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs @@ -5,7 +5,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; /// The project directory, defaulting to the working directory, with its inputs and adapter. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs index e180f53fa8..1ac9d93180 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs @@ -13,7 +13,7 @@ const ADOPTION_TYPES_MODULE: &str = "sqlbuild.compiler.contract_adoption.types"; const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const TYPES_EQUAL_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; -const DYNAMIC_FAMILIES_MODULE: &str = "sqlbuild.spec.contracts.main.matching_dynamic_families"; +const DYNAMIC_FAMILIES_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; const MODEL_RESOURCE: &str = "model"; const SOURCE_RESOURCE: &str = "source"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs index 8cfc8ae1cc..09dc56013d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs @@ -33,6 +33,7 @@ use sqlbuild_refactor::refactoring::models::{ use sqlbuild_refactor::refactoring::types::Originals; use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_guard}; +use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::lineage::lineage_index::lineage_key; use crate::bindings::_helpers::refactor::refactor_facts::{ declaration_placement, model_facts, refactor_facts, @@ -41,7 +42,6 @@ use crate::bindings::_helpers::refactor::refactor_facts::{ const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const REFACTOR_ERRORS_MODULE: &str = "sqlbuild.compiler.refactoring.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const STATUS_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; const COMPILE_MODULE: &str = "sqlbuild.cli.commands._helpers.refactor.compile_facts"; const STAGING_PREFIX: &str = "sqb-refactor-"; const MOVE_COMMAND: &str = "mv"; @@ -430,12 +430,8 @@ fn run_refactor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult< let stream: Bound<'_, PyAny> = py.import("sys")? .getattr(if json_output { "stderr" } else { "stdout" })?; - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("stream", stream)?; - options.set_item("use_color", use_color)?; let status: Status<'_> = Status { - reporter: imported(py, STATUS_MODULE, "TransientStatusReporter")? - .call((), Some(&options))?, + reporter: status_reporter(&stream, use_color)?, }; let result: PyResult> = compiler_guard(|| run(py, &command, &status)); status.close()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs index 18b161d1d6..205b8f1d28 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs @@ -2,8 +2,8 @@ use std::collections::HashMap; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use sqlbuild_refactor::refactoring::models::{ AuthoredFile, DeclarationPlacement, DependencyFacts, DestinationFacts, DiscoveredFile, ExpansionFacts, ExpansionSpan, LocationFacts, ManualLocation, ModelFacts, PlacedDeclaration, @@ -11,6 +11,10 @@ use sqlbuild_refactor::refactoring::models::{ SchemaColumnFacts, SqlFileRole, }; +use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; +use crate::bindings::_helpers::scopes::scope_reports::scope_move_preview; +use crate::bindings::_helpers::scopes::scope_validation::scope_relocated_declarations; + const NATIVE_MODULE: &str = "sqlbuild._native"; const CONFIG_MODULE: &str = "sqlbuild._native"; const SCOPE_MODELS: &str = "sqlbuild.compiler.scopes.models"; @@ -317,11 +321,14 @@ pub(crate) fn declaration_placement( .getattr("graph")? .getattr("project")? .getattr("scope_index")?; - let lookup: Bound<'_, PyAny> = native(py, "build_scope_lookup")?.call1((&index,))?; - let (preview, diagnostics): (Bound<'_, PyAny>, Bound<'_, PyAny>) = - native(py, "scope_move_preview")? - .call1((lookup, format!("{MODEL_KIND_PREFIX}{model}"), destination))? - .extract()?; + let lookup: Bound<'_, PyAny> = build_scope_lookup(py, &index)?.into_bound(py); + let (preview, diagnostics) = scope_move_preview( + py, + &lookup, + &PyString::new(py, &format!("{MODEL_KIND_PREFIX}{model}")).into_any(), + &PyString::new(py, destination).into_any(), + )?; + let diagnostics: Bound<'_, PyAny> = diagnostics.into_bound(py); let relocated: Option> = if preview.is_none() { None } else { @@ -337,11 +344,10 @@ pub(crate) fn declaration_placement( .import(SCOPE_MODELS)? .getattr("ResourceIdentity")? .call((), Some(&options))?; - let records: Bound<'_, PyAny> = - native(py, "scope_relocated_declarations")?.call1((&index, resource, destination))?; - if records.is_none() { - None - } else { + let records: Option> = + scope_relocated_declarations(py, &index, &resource, destination)?; + if let Some(records) = records { + let records: Bound<'_, PyAny> = records.into_bound(py).into_any(); let sort: Bound<'_, PyDict> = PyDict::new(py); sort.set_item( "key", @@ -363,6 +369,8 @@ pub(crate) fn declaration_placement( }); } Some(relocated) + } else { + None } }; let mut declarations: Vec = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs new file mode 100644 index 0000000000..05dfc3df4a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs @@ -0,0 +1,306 @@ +//! Authored SQL expanded through project vars, declarations and macros, with each pass's spans. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::model_config::template_data::record_reads; +use crate::bindings::_helpers::render::declaration_expansion::{ + DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, SpanRow, + expand_declarations, span_rows, +}; +use crate::bindings::_helpers::render::declaration_values::resolved_declaration_context; +use crate::bindings::_helpers::render::macro_expansion_result::expand_sql_macros; +use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; +use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, interpolation_batch}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; +const FUNCTOOLS_MODULE: &str = "functools"; +const DECLARATION_FIELDS: [&str; 4] = [ + "enums", + "constants", + "inaccessible_enums", + "inaccessible_constants", +]; + +/// `sql` with `@@` tokens interpolated and each substitution's span, raising its error. +fn substituted<'py>( + sql: &Bound<'py, PyAny>, + file_path: &Bound<'py, PyAny>, + (effective_vars, context_values): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult<(String, Vec)> { + let py: Python<'py> = sql.py(); + let sources = InterpolationSources( + effective_vars.downcast()?.clone(), + imported(py, UNICODE_ENVIRONMENT_MODULE, "UnicodeEnvironment")?.call0()?, + if context_values.is_none() { + None + } else { + Some(context_values.downcast()?.clone()) + }, + imported(py, NATIVE_MODULE, "render_project_var_text")?, + ); + let text: String = sql.extract()?; + let path: String = file_path.str()?.extract()?; + let rows = interpolation_batch(sources, &[(text.clone(), path)])?; + let Some((rendered, spans, reads, error)) = rows.into_iter().next() else { + return Ok((text, Vec::new())); + }; + record_reads(py, reads.iter().map(|(kind, name)| (*kind, name.as_str())))?; + if let Some(message) = error { + return Err(PyErr::from_value( + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?.call1((message,))?, + )); + } + Ok((rendered.unwrap_or(text), spans)) +} + +/// `ExpansionSpan` objects for span rows, in order. +fn span_objects<'py>(py: Python<'py>, spans: &[SpanRow]) -> PyResult> { + let span_class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS_MODULE, "ExpansionSpan")?; + let objects: Vec> = spans + .iter() + .map(|span| span_class.call1(*span)) + .collect::>()?; + PyTuple::new(py, objects) +} + +/// The authored SQL's public declaration context: visible and inaccessible enums and constants. +fn declaration_context<'py>( + py: Python<'py>, + declarations: &Bound<'py, PyAny>, +) -> PyResult> { + let fields: Vec<(&str, Bound<'py, PyAny>)> = DECLARATION_FIELDS + .iter() + .map(|field| { + Ok(( + *field, + if declarations.is_none() { + PyDict::new(py).into_any() + } else { + declarations.getattr(*field)? + }, + )) + }) + .collect::>()?; + call_with( + &imported(py, COMPILE_MODELS_MODULE, "DeclarationResolutionContext")?, + &fields, + ) +} + +/// `sql` with natively scanned enum and constant references spliced, and their spans. +fn declarations_expanded<'py>( + sql: &str, + file_path: &Bound<'py, PyAny>, + (declarations, value_renderer, collection_rendering): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult<(String, Vec)> { + let py: Python<'py> = file_path.py(); + let (references, stop) = declaration_scan(sql); + if references.is_empty() && stop.is_none() { + return Ok((sql.to_owned(), Vec::new())); + } + let context: Bound<'py, PyAny> = declaration_context(py, declarations)?; + let render_constant: Bound<'py, PyAny> = imported(py, FUNCTOOLS_MODULE, "partial")?.call( + (imported(py, NATIVE_MODULE, "render_constant_declaration")?,), + Some(&keywords( + py, + &[ + ("value_renderer", value_renderer.clone()), + ("collection_rendering", collection_rendering.clone()), + ("file_path", file_path.clone()), + ], + )?), + )?; + let classes = ExpansionClasses { + usage_record: imported(py, SCOPE_MODELS_MODULE, "UsageRecord")?, + compile_error: imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?, + }; + let path: String = file_path.str()?.extract()?; + let expanded: ExpandedDeclarations<'py> = expand_declarations(&DeclarationExpansion { + sql, + references: &references, + stop, + context: &context, + consumer: ExpansionConsumer { + identity: None, + relationship_visibility: false, + path: &path, + }, + render_constant: &render_constant, + classes: &classes, + })?; + let spans: Vec = span_rows(&expanded.spliced); + Ok((expanded.spliced.sql, spans)) +} + +/// Expand authored SQL, returning each pass's substitution spans in order. +fn authored_sql_with_spans<'py>(request: &Request<'_, 'py>) -> PyResult> { + let py: Python<'py> = request.py(); + let none = || py.None().into_bound(py); + let file_path: Bound<'py, PyAny> = request.get("file_path")?; + let (interpolated, interpolation_spans) = substituted( + &request.get("sql")?, + &file_path, + ( + &request.get("effective_vars")?, + &request.optional("context_values")?.unwrap_or_else(none), + ), + )?; + let (declared, declaration_spans) = declarations_expanded( + &interpolated, + &file_path, + ( + &request.optional("declarations")?.unwrap_or_else(none), + &request.get("value_renderer")?, + &request.get("collection_rendering")?, + ), + )?; + let macro_options: Bound<'py, PyDict> = keywords( + py, + &[ + ("sql", PyString::new(py, &declared).into_any()), + ("file_path", file_path), + ("loaded_macros", request.get("loaded_macros")?), + ("macro_context", request.get("macro_context")?), + ( + "declaration_resolver", + request + .optional("declaration_resolver")? + .unwrap_or_else(none), + ), + ], + )?; + let expansion: Bound<'py, PyAny> = expand_sql_macros(py, Some(macro_options))?; + let passes: Bound<'py, PyTuple> = PyTuple::new( + py, + [ + span_objects(py, &interpolation_spans)?.into_any(), + span_objects(py, &declaration_spans)?.into_any(), + expansion.getattr("spans")?, + ], + )?; + PyTuple::new(py, [expansion.getattr("sql")?, passes.into_any()]) +} + +/// Expand authored SQL, returning each pass's substitution spans in order. +#[pyfunction] +#[pyo3(signature = (**request))] +fn expand_authored_sql_with_spans<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + authored_sql_with_spans(&Request::new("expand_authored_sql_with_spans", request)?) +} + +/// The declarations visible to `file_path`: the resolver's, else the context's, plus model-local. +fn path_declarations<'py>( + context: &Bound<'py, PyAny>, + file_path: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = context.py(); + let resolution_class: Bound<'py, PyAny> = + imported(py, COMPILE_MODELS_MODULE, "DeclarationResolutionContext")?; + let resolver: Bound<'py, PyAny> = context.getattr("declaration_resolver")?; + let declarations: Bound<'py, PyAny> = if resolver.is_none() { + call_with( + &resolution_class, + &[ + ("enums", context.getattr("enums")?), + ("constants", context.getattr("constants")?), + ], + )? + } else { + resolved_declaration_context(&resolver, file_path, None)? + }; + let local: Bound<'py, PyAny> = context + .getattr("local_declarations")? + .call_method1("get", (file_path,))?; + if local.is_none() { + return Ok(declarations); + } + let merged = |field: &str| -> PyResult> { + declarations.getattr(field)?.bitor(local.getattr(field)?) + }; + call_with( + &resolution_class, + &[ + ("enums", merged("enums")?), + ("constants", merged("constants")?), + ( + "inaccessible_enums", + declarations.getattr("inaccessible_enums")?, + ), + ( + "inaccessible_constants", + declarations.getattr("inaccessible_constants")?, + ), + ], + ) +} + +/// Expand authored SQL with a project's expansion context and report each pass's spans. +#[pyfunction] +#[pyo3(signature = (*, sql, file_path, context))] +fn expand_sql_with_spans<'py>( + sql: &Bound<'py, PyAny>, + file_path: &Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = sql.py(); + let arguments: Bound<'py, PyDict> = keywords( + py, + &[ + ("sql", sql.clone()), + ("file_path", file_path.clone()), + ("effective_vars", context.getattr("effective_vars")?), + ("loaded_macros", context.getattr("loaded_macros")?), + ("macro_context", context.getattr("macro_context")?), + ("declarations", path_declarations(context, file_path)?), + ( + "declaration_resolver", + context.getattr("declaration_resolver")?, + ), + ("value_renderer", context.getattr("value_renderer")?), + ( + "collection_rendering", + context.getattr("collection_rendering")?, + ), + ], + )?; + authored_sql_with_spans(&Request::new("expand_sql_with_spans", Some(&arguments))?) +} + +/// Replace SQL interpolation tokens, returning the span of every substitution. +#[pyfunction] +#[pyo3(signature = (*, sql, file_path, effective_vars, context_values=None))] +fn substitute_sql_vars_with_spans<'py>( + sql: &Bound<'py, PyAny>, + file_path: &Bound<'py, PyAny>, + effective_vars: &Bound<'py, PyAny>, + context_values: Option>, +) -> PyResult<(String, Bound<'py, PyTuple>)> { + let py: Python<'py> = sql.py(); + let context_values: Bound<'py, PyAny> = + context_values.unwrap_or_else(|| py.None().into_bound(py)); + let (substituted_sql, spans) = substituted(sql, file_path, (effective_vars, &context_values))?; + Ok((substituted_sql, span_objects(py, &spans)?)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(expand_authored_sql_with_spans, module)?)?; + module.add_function(wrap_pyfunction!(expand_sql_with_spans, module)?)?; + module.add_function(wrap_pyfunction!(substitute_sql_vars_with_spans, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs index 8cfb32f4c2..599a082a4d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs @@ -1,8 +1,8 @@ //! Enum and constant references expanded natively, for model queries and every other caller. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyInt, PyTuple}; -use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::errors::DeclarationError; use sqlbuild_analysis::compiled_project::main::declaration_error_message::declaration_error_message; use sqlbuild_analysis::compiled_project::main::enum_member_literal::enum_member_literal; @@ -292,35 +292,3 @@ pub(crate) fn span_rows(spliced: &SplicedSql) -> Vec { }) .collect() } - -/// Expand one string's scanned references; `source` is `(sql, references, stop code)`. -#[pyfunction] -fn expand_declaration_references<'py>( - source: (String, Vec, Option), - context: Bound<'py, PyAny>, - consumer: (Option>, bool, String), - renderers: (Bound<'py, PyAny>, ExpansionClasses<'py>), -) -> PyResult<(String, Vec, Vec>)> { - let (sql, references, stop) = source; - let (identity, relationship_visibility, path) = consumer; - let (render_constant, classes) = renderers; - let expanded: ExpandedDeclarations<'py> = expand_declarations(&DeclarationExpansion { - sql: &sql, - references: &references, - stop, - context: &context, - consumer: ExpansionConsumer { - identity, - relationship_visibility, - path: &path, - }, - render_constant: &render_constant, - classes: &classes, - })?; - let spans: Vec = span_rows(&expanded.spliced); - Ok((expanded.spliced.sql, spans, expanded.usages)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(expand_declaration_references, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs new file mode 100644 index 0000000000..d3972c62b6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs @@ -0,0 +1,472 @@ +//! Declaration scope resolvers, per-path declaration contexts and adapter-rendered constants. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyInt, PyList, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const NATIVE_MODULE: &str = "sqlbuild._native"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const BUILTINS_MODULE: &str = "builtins"; +const NO_DECLARATION_KIND: u8 = 255; +/// Declaration kinds by their native code: enum 0, constant 1, macro 2. +const DECLARATION_KINDS: [&str; 3] = ["ENUM", "CONSTANT", "MACRO"]; +const SQL_VALUE_ERRORS_MODULE: &str = "sqlbuild.sql_values.exceptions"; +const TYPES_MODULE: &str = "types"; +const SCALAR_KINDS: [&str; 6] = ["string", "integer", "boolean", "float", "decimal", "null"]; +const COLLECTION_KINDS: [&str; 2] = ["list", "set"]; +const ARRAY_RENDERING: &str = "array"; +const MAX_SQL_VALUE_SIZE: usize = 1_000_000; + +/// `(kind, name, owner)` identity of one declaration, built as `DeclarationIdentity`. +fn declaration_identity<'py>( + classes: &(Bound<'py, PyAny>, Bound<'py, PyAny>), + kind: &str, + name: &Bound<'py, PyAny>, + owner: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let (identity_class, kinds) = classes; + let kind: Bound<'py, PyAny> = kinds.getattr(kind)?; + match owner { + Some(owner) => identity_class.call1((kind, name, owner)), + None => identity_class.call1((kind, name)), + } +} + +/// Each project declaration's identity mapped to its process-local value. +fn runtime_declarations<'py>( + discovered: &Bound<'py, PyAny>, + loaded_macros: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let classes = ( + imported(py, SCOPE_MODELS_MODULE, "DeclarationIdentity")?, + imported(py, SCOPE_TYPES_MODULE, "DeclarationKind")?, + ); + let declarations: Bound<'py, PyDict> = PyDict::new(py); + for (files, kind) in [("enum_files", "ENUM"), ("constant_files", "CONSTANT")] { + for file in discovered.getattr(files)?.try_iter()? { + for declaration in file?.getattr("declarations")?.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + let identity = + declaration_identity(&classes, kind, &declaration.getattr("name")?, None)?; + declarations.set_item(identity, declaration)?; + } + } + } + let resource_identity: Bound<'py, PyAny> = + imported(py, SCOPE_MODELS_MODULE, "ResourceIdentity")?; + let model_kind: Bound<'py, PyAny> = + imported(py, SCOPE_TYPES_MODULE, "ResourceKind")?.getattr("MODEL")?; + for model_file in discovered.getattr("model_files")?.try_iter()? { + let model_file: Bound<'py, PyAny> = model_file?; + let name: Bound<'py, PyAny> = model_file + .getattr("header_values")? + .call_method1("get", ("name",))?; + let name: Bound<'py, PyAny> = if name.is_instance_of::() && name.is_truthy()? { + name + } else { + model_file.getattr("relative_path")?.getattr("stem")? + }; + let owner: Bound<'py, PyAny> = resource_identity.call1((&model_kind, name))?; + for (attribute, kind) in [ + ("enum_declarations", "ENUM"), + ("constant_declarations", "CONSTANT"), + ] { + for declaration in model_file.getattr(attribute)?.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + let identity = declaration_identity( + &classes, + kind, + &declaration.getattr("name")?, + Some(&owner), + )?; + declarations.set_item(identity, declaration)?; + } + } + } + if !loaded_macros.is_none() { + for name_and_macro in loaded_macros.call_method0("values")?.try_iter()? { + let declared_macro: Bound<'py, PyAny> = name_and_macro?; + let identity = + declaration_identity(&classes, "MACRO", &declared_macro.getattr("name")?, None)?; + declarations.set_item(identity, declared_macro)?; + } + } + Ok(declarations) +} + +/// Resources whose declarations differ from their directory's: private owners and grantees. +fn resource_specific<'py>(scope_index: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = scope_index.py(); + let private: Bound<'py, PyAny> = + imported(py, SCOPE_TYPES_MODULE, "ScopeKind")?.getattr("PRIVATE")?; + let mut resources: Vec> = Vec::new(); + for declaration in scope_index.getattr("declarations")?.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + let owner: Bound<'py, PyAny> = declaration.getattr("identity")?.getattr("owner")?; + if declaration.getattr("scope")?.is(&private) && !owner.is_none() { + resources.push(owner); + } + } + for grant in scope_index.getattr("grants")?.try_iter()? { + resources.push(grant?.getattr("resource")?); + } + PyFrozenSet::new(py, resources) +} + +/// Pair the serializable static index with original process-local declaration values. +pub(crate) fn declaration_scope_resolver<'py>( + discovered: &Bound<'py, PyAny>, + scope_index: &Bound<'py, PyAny>, + (loaded_macros, lookup): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let declarations: Bound<'py, PyDict> = runtime_declarations(discovered, loaded_macros)?; + let lookup: Bound<'py, PyAny> = if lookup.is_none() { + build_scope_lookup(py, scope_index)?.into_bound(py) + } else { + lookup.clone() + }; + let projection: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "DeclarationRuntimeProjection")?, + &[( + "declarations", + imported(py, TYPES_MODULE, "MappingProxyType")?.call1((&declarations,))?, + )], + )?; + let native_contexts: Bound<'py, PyAny> = + native_declaration_contexts(&lookup, declarations.as_any())?; + call_with( + &imported(py, COMPILE_MODELS_MODULE, "DeclarationScopeResolver")?, + &[ + ("project_dir", discovered.getattr("project_dir")?), + ("lookup", lookup), + ("projection", projection), + ( + "resource_specific", + resource_specific(scope_index)?.into_any(), + ), + ("native_contexts", native_contexts), + ], + ) +} + +/// The native code of declaration kind `kind`, or the no-kind code. +fn kind_code(kinds: &[Bound<'_, PyAny>], kind: &Bound<'_, PyAny>) -> u8 { + for (code, known) in (0_u8..).zip(kinds.iter()) { + if known.is(kind) { + return code; + } + } + NO_DECLARATION_KIND +} + +/// The native kind code of a runtime declaration value, by its class. +fn value_kind_code(classes: &[Bound<'_, PyAny>], value: &Bound<'_, PyAny>) -> PyResult { + for (code, class) in (0_u8..).zip(classes.iter()) { + if value.is_instance(class)? { + return Ok(code); + } + } + Ok(NO_DECLARATION_KIND) +} + +/// `{owner: list(items)}` of an index position mapping. +fn listed_positions<'py>(positions: &Bound<'py, PyAny>) -> PyResult> { + let listed: Bound<'py, PyDict> = PyDict::new(positions.py()); + for item in positions.call_method0("items")?.try_iter()? { + let (owner, items): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + listed.set_item( + owner, + PyList::new( + positions.py(), + items.try_iter()?.collect::>>()?, + )?, + )?; + } + Ok(listed) +} + +/// Hand the lookup's declaration positions and runtime values to the native classifier. +pub(crate) fn native_declaration_contexts<'py>( + lookup: &Bound<'py, PyAny>, + declarations: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = lookup.py(); + let kind_type: Bound<'py, PyAny> = imported(py, SCOPE_TYPES_MODULE, "DeclarationKind")?; + let kinds: Vec> = DECLARATION_KINDS + .iter() + .map(|kind| kind_type.getattr(*kind)) + .collect::>()?; + let value_classes: Vec> = vec![ + imported(py, DISCOVERY_MODELS_MODULE, "EnumDeclaration")?, + imported(py, DISCOVERY_MODELS_MODULE, "ConstantDeclaration")?, + imported(py, COMPILE_MODELS_MODULE, "LoadedMacro")?, + ]; + let index: Bound<'py, PyAny> = lookup.getattr("visibility_index")?; + let lookup_declarations: Bound<'py, PyAny> = lookup.getattr("declarations")?; + let identity_keys: Bound<'py, PyDict> = PyDict::new(py); + let mut positions: Vec> = Vec::new(); + for identity in index.getattr("identities")?.try_iter()? { + let identity: Bound<'py, PyAny> = identity?; + let key: Bound<'py, PyAny> = + identity_keys.call_method1("setdefault", (&identity, identity_keys.len()))?; + let value: Bound<'py, PyAny> = declarations.call_method1("get", (&identity,))?; + positions.push(PyTuple::new( + py, + [ + identity.clone(), + lookup_declarations.get_item(&identity)?.get_item(0)?, + identity.getattr("name")?, + kind_code(&kinds, &identity.getattr("kind")?) + .into_pyobject(py)? + .into_any(), + value.clone(), + value_kind_code(&value_classes, &value)? + .into_pyobject(py)? + .into_any(), + key, + ], + )?); + } + let reasons: Bound<'py, PyDict> = PyDict::new(py); + for reason in imported(py, SCOPE_TYPES_MODULE, "VisibilityReason")?.try_iter()? { + let reason: Bound<'py, PyAny> = reason?; + reasons.set_item(reason.getattr("value")?, reason)?; + } + let classes: Bound<'py, PyDict> = PyDict::new(py); + classes.set_item( + "visibility_record", + imported(py, SCOPE_MODELS_MODULE, "VisibilityRecord")?, + )?; + classes.set_item( + "resolution_context", + imported(py, COMPILE_MODELS_MODULE, "DeclarationResolutionContext")?, + )?; + classes.set_item( + "allocate", + imported(py, BUILTINS_MODULE, "object")?.getattr("__new__")?, + )?; + classes.set_item("reasons", reasons)?; + let keys: Bound<'py, PyDict> = PyDict::new(py); + keys.set_item("identity_keys", identity_keys)?; + keys.set_item("private_positions", index.getattr("private_positions")?)?; + keys.set_item("grants_by_resource", lookup.getattr("grants_by_resource")?)?; + let lookups: Bound<'py, PyDict> = PyDict::new(py); + lookups.set_item("resources_by_path", lookup.getattr("resources_by_path")?)?; + lookups.set_item("resources", lookup.getattr("resources")?)?; + lookups.set_item("declarations", &lookup_declarations)?; + lookups.set_item( + "index_declarations", + lookup.getattr("index")?.getattr("declarations")?, + )?; + for (name, class) in [ + ("resource_record", "ResourceRecord"), + ("ownership_root", "OwnershipRoot"), + ("resource_identity", "ResourceIdentity"), + ("declaration_identity", "DeclarationIdentity"), + ] { + lookups.set_item(name, imported(py, SCOPE_MODELS_MODULE, class)?)?; + } + lookups.set_item( + "resource_kind", + imported(py, SCOPE_TYPES_MODULE, "ResourceKind")?, + )?; + lookups.set_item("declaration_kind", kind_type)?; + lookups.set_item( + "private_scope", + imported(py, SCOPE_TYPES_MODULE, "ScopeKind")?.getattr("PRIVATE")?, + )?; + lookups.set_item( + "identity_error", + imported(py, SCOPE_ERRORS_MODULE, "InvalidQualifiedIdentityError")?, + )?; + lookups.set_item( + "path_error", + imported(py, SCOPE_ERRORS_MODULE, "InvalidScopePathError")?, + )?; + let global_positions: Bound<'py, PyList> = PyList::new( + py, + index + .getattr("global_positions")? + .try_iter()? + .collect::>>()?, + )?; + imported(py, NATIVE_MODULE, "NativeDeclarationContexts")?.call1(( + PyList::new(py, positions)?, + ( + global_positions, + listed_positions(&index.getattr("local_positions")?)?, + listed_positions(&index.getattr("inherited_positions")?)?, + ), + classes, + (keys, lookups), + )) +} + +/// Hand one scope lookup's positions and runtime values to the native classifier. +#[pyfunction] +#[pyo3(signature = (*, lookup, declarations))] +fn build_native_declaration_contexts<'py>( + lookup: &Bound<'py, PyAny>, + declarations: &Bound<'py, PyAny>, +) -> PyResult> { + native_declaration_contexts(lookup, declarations) +} + +/// Canonical visibility for one authored path, projected onto runtime declaration values. +pub(crate) fn resolved_declaration_context<'py>( + resolver: &Bound<'py, PyAny>, + file_path: &Bound<'py, PyAny>, + resource: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let py: Python<'py> = resolver.py(); + let resource: Bound<'py, PyAny> = + resource.map_or_else(|| py.None().into_bound(py), Clone::clone); + resolver + .getattr("native_contexts")? + .call_method1("resolve", (resolver, file_path, resource))? + .get_item(0) +} + +/// The adapter's SQL for one constant under `selected` collection rendering, or its error. +fn rendered_constant<'py>( + declaration: &Bound<'py, PyAny>, + value_renderer: &Bound<'py, PyAny>, + selected: &Bound<'py, PyAny>, +) -> PyResult { + let value: Bound<'py, PyAny> = declaration.getattr("value")?; + let kind: String = value.getattr("kind")?.str()?.extract()?; + let method: &str = if SCALAR_KINDS.contains(&kind.as_str()) { + "render_typed_scalar" + } else if COLLECTION_KINDS.contains(&kind.as_str()) { + if selected.eq(ARRAY_RENDERING)? { + "render_typed_array" + } else { + "render_typed_value_list" + } + } else { + "render_typed_object" + }; + let rendered: String = + call_with(&value_renderer.getattr(method)?, &[("value", value)])?.extract()?; + let size: usize = rendered.len(); + if size > MAX_SQL_VALUE_SIZE { + let context: String = format!( + "{} constant '{}'", + declaration.getattr("relative_path")?.str()?, + declaration.getattr("name")?.str()? + ); + return Err(PyErr::from_value( + imported( + declaration.py(), + SQL_VALUE_ERRORS_MODULE, + "SqlValueValidationError", + )? + .call1((format!( + "{context} rendered value is {size} bytes; maximum is {MAX_SQL_VALUE_SIZE} bytes" + ),))?, + )); + } + Ok(rendered) +} + +/// Render one validated constant with the active adapter's typed-value contract. +pub(crate) fn constant_declaration_sql<'py>( + declaration: &Bound<'py, PyAny>, + value_renderer: &Bound<'py, PyAny>, + (collection_rendering, file_path): (&Bound<'py, PyAny>, Option<&Bound<'py, PyAny>>), +) -> PyResult { + let py: Python<'py> = declaration.py(); + let render_as: Bound<'py, PyAny> = declaration.getattr("render_as")?; + let selected: Bound<'py, PyAny> = if render_as.is_truthy()? { + render_as + } else { + collection_rendering.clone() + }; + let error: PyErr = match rendered_constant(declaration, value_renderer, &selected) { + Ok(rendered) => return Ok(rendered), + Err(error) => error, + }; + let rendering_error: Bound<'py, PyAny> = + imported(py, SQL_VALUE_ERRORS_MODULE, "SqlValueRenderingError")?; + let validation_error: Bound<'py, PyAny> = + imported(py, SQL_VALUE_ERRORS_MODULE, "SqlValueValidationError")?; + let raised: Bound<'py, PyAny> = error.value(py).clone().into_any(); + if !(raised.is_instance(&rendering_error)? || raised.is_instance(&validation_error)?) { + return Err(error); + } + let relative_path: Bound<'py, PyAny> = declaration.getattr("relative_path")?; + let location: Bound<'py, PyAny> = match file_path { + Some(file_path) if !file_path.is_none() => file_path.clone(), + _ => relative_path.clone(), + }; + let compile_error: Bound<'py, PyAny> = + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?.call1((format!( + "{} constant '{}' could not be rendered in '{}' by adapter '{}' as {}: {}", + relative_path.str()?, + declaration.getattr("name")?.str()?, + location.str()?, + value_renderer.getattr("adapter_name")?.str()?, + selected.getattr("value")?.str()?, + raised.str()?, + ),))?; + compile_error.setattr("__cause__", &raised)?; + Err(PyErr::from_value(compile_error)) +} + +/// Render one validated constant with the active adapter's typed-value contract. +#[pyfunction] +#[pyo3(signature = (*, declaration, value_renderer, collection_rendering, file_path=None))] +fn render_constant_declaration<'py>( + declaration: &Bound<'py, PyAny>, + value_renderer: &Bound<'py, PyAny>, + collection_rendering: &Bound<'py, PyAny>, + file_path: Option>, +) -> PyResult { + constant_declaration_sql( + declaration, + value_renderer, + (collection_rendering, file_path.as_ref()), + ) +} + +/// One enum member value as a SQL scalar literal. +#[pyfunction] +#[pyo3(signature = (*, value))] +fn render_enum_member_value(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_instance_of::() { + return value.str()?.extract(); + } + let text: String = value.extract()?; + Ok(format!("'{}'", text.replace('\'', "''"))) +} + +/// Canonical visibility for one authored path, projected onto runtime declaration values. +#[pyfunction] +#[pyo3(signature = (*, resolver, file_path, resource=None))] +fn resolve_declaration_context<'py>( + resolver: &Bound<'py, PyAny>, + file_path: &Bound<'py, PyAny>, + resource: Option>, +) -> PyResult> { + resolved_declaration_context(resolver, file_path, resource.as_ref()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_native_declaration_contexts, module)?)?; + module.add_function(wrap_pyfunction!(render_constant_declaration, module)?)?; + module.add_function(wrap_pyfunction!(render_enum_member_value, module)?)?; + module.add_function(wrap_pyfunction!(resolve_declaration_context, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index 1137cc93cb..d6fe88b2b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -9,6 +9,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; +use crate::bindings::_helpers::render::declaration_values::resolved_declaration_context; use crate::bindings::_helpers::render::macro_call_rows::span_row; use crate::bindings::_helpers::render::macro_expansion::{ ExpansionOutput, NativeMacroExpansion, expanded, @@ -24,7 +25,6 @@ const REF_MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; const REF_TYPES_MODULE: &str = "sqlbuild.python_nodes.types"; const MACRO_ARGUMENTS_MODULE: &str = "sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments"; -const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; const USER_MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; const ACTIVE_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; const BRIDGE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.macro_bridge.constants"; @@ -204,13 +204,9 @@ fn expansion_result<'py>(request: ExpansionRequest<'py>) -> PyResult = match (request.declarations, request.declaration_resolver) { (Some(declarations), _) => declarations, - (None, Some(resolver)) => call_with( - &imported(py, DECLARATIONS_MODULE, "resolve_declaration_context")?, - &[ - ("resolver", resolver), - ("file_path", request.file_path.clone()), - ], - )?, + (None, Some(resolver)) => { + resolved_declaration_context(&resolver, &request.file_path, None)? + } (None, None) => py.None().into_bound(py), }; let expansion: NativeMacroExpansion = NativeMacroExpansion::new( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs index 84bb516f80..f678671194 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs @@ -1,5 +1,7 @@ +pub(crate) mod authored_sql; pub(crate) mod declaration_expansion; pub(crate) mod declaration_indexes; +pub(crate) mod declaration_values; pub(crate) mod macro_arguments; pub(crate) mod macro_call_classes; pub(crate) mod macro_call_memo; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs index 2ea4348538..5f05957dfc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs @@ -3,12 +3,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::render::{ - declaration_expansion, declaration_indexes, macro_arguments, macro_calls, macro_expansion, - macro_expansion_result, macro_modules, + authored_sql, declaration_indexes, declaration_values, macro_arguments, macro_calls, + macro_expansion, macro_expansion_result, macro_modules, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - declaration_expansion::register(module)?; + authored_sql::register(module)?; + declaration_values::register(module)?; declaration_indexes::register(module)?; macro_arguments::register(module)?; macro_calls::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs index 88fd9ad7aa..e7f3325b56 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs @@ -14,7 +14,7 @@ const PYTHON_NODE_MODELS: &str = "sqlbuild.executor.python_nodes.models"; const TESTING_MODELS: &str = "sqlbuild.executor.testing.models"; const AUDITING_MODELS: &str = "sqlbuild.executor.auditing.models"; const CLONE_MODELS: &str = "sqlbuild.executor.clone.models"; -const SQL_TEST_ID_MODULE: &str = "sqlbuild.executor.testing.main.resource_id"; +const SQL_TEST_ID_MODULE: &str = "sqlbuild.executor.testing.main._resource_id"; const AUDIT_ID_MODULE: &str = "sqlbuild.executor.auditing.main.resource_id"; /// The executor result types that result output distinguishes. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs index 30ab858b7a..29ec865df7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs @@ -2,9 +2,8 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_rules::rules::main::selected_codes; -use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; +use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::types::CompilerDetach; #[pyfunction] @@ -15,13 +14,7 @@ fn run_custom_host_json(py: Python<'_>, spec_json: &str) -> PyResult { .map_err(value_error) } -#[pyfunction] -fn selected_codes_json(request_json: &str) -> PyResult { - compiler_guard(|| selected_codes::selected_codes_json(request_json).map_err(value_error)) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(run_custom_host_json, module)?)?; - module.add_function(wrap_pyfunction!(selected_codes_json, module)?)?; crate::bindings::_helpers::rules::rows::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs new file mode 100644 index 0000000000..59eac5be48 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs @@ -0,0 +1,128 @@ +//! The SQL expansion context compiler-integrated lint expands authored bodies with. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::build_sql_expansion_context; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.adapter.discovery.main.resolve_adapter"; +const ADAPTER_EXCEPTIONS_MODULE: &str = "sqlbuild.adapter.contract.exceptions"; +const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const DISCOVERY_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild.lint.exceptions"; +const COMPILE_FIRST_MESSAGE: &str = "compiler-integrated Rules check the SQL your project \ + actually produces, so the project must compile first"; + +/// Build the lint expansion context, reporting compile failures as `ProjectCompileError`. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_lint_expansion_context<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_lint_expansion_context", + options, + ( + &["project_dir"], + &[ + "value_renderer", + "discovered_inputs", + "declaration_scope", + "static_declaration_scope", + ], + ), + )?; + expansion_context(py, &arguments).or_else(|error| Err(compile_first_error(py, error)?)) +} + +fn expansion_context<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, +) -> PyResult> { + let project_dir: Bound<'py, PyAny> = arguments.value("project_dir")?; + let mut discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; + if discovered.is_none() { + let no: Bound<'py, PyAny> = PyBool::new(py, false).to_owned().into_any(); + discovered = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[ + ("project_dir", project_dir.clone()), + ("sql_analysis_enabled_override", no.clone()), + ("extract_output_column_locations", no), + ], + )?; + } + let mut renderer: Bound<'py, PyAny> = arguments.value("value_renderer")?; + if !renderer.is_truthy()? { + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + renderer = call_with( + &imported(py, RESOLVE_ADAPTER_MODULE, "resolve_adapter")?, + &[ + ("adapter_name", adapter_name), + ("project_dir", project_dir.clone()), + ], + )?; + } + build_sql_expansion_context( + py, + Some(keywords( + py, + &[ + ("project_dir", project_dir), + ("discovered_inputs", discovered), + ("value_renderer", renderer), + ("declaration_scope", arguments.value("declaration_scope")?), + ( + "static_declaration_scope", + arguments.value("static_declaration_scope")?, + ), + ], + )?), + ) +} + +/// Adapter, compile and discovery errors as `ProjectCompileError`; any other error unchanged. +fn compile_first_error(py: Python<'_>, error: PyErr) -> PyResult { + let value: Bound<'_, PyAny> = error.value(py).clone().into_any(); + let reported: bool = value.is_instance(&imported( + py, + ADAPTER_EXCEPTIONS_MODULE, + "AdapterUserError", + )?)? || value.is_instance(&imported( + py, + COMPILE_EXCEPTIONS_MODULE, + "CompileInputError", + )?)? || value.is_instance(&imported( + py, + DISCOVERY_EXCEPTIONS_MODULE, + "DiscoveryError", + )?)?; + if !reported { + return Ok(error); + } + let message: String = format!("{COMPILE_FIRST_MESSAGE}: {}", value.str()?); + let raised: PyErr = PyErr::from_value( + imported(py, LINT_EXCEPTIONS_MODULE, "ProjectCompileError")?.call1((message,))?, + ); + raised.set_cause(py, Some(error)); + Ok(raised) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_lint_expansion_context, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 3958390e03..5bad746aca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod dsl_headers; pub(crate) mod evaluation; pub(crate) mod findings; pub(crate) mod lint; +pub(crate) mod lint_expansion; pub(crate) mod lint_sql_bodies; pub(crate) mod lint_text; pub(crate) mod registration; @@ -14,5 +15,6 @@ pub(crate) mod rows; pub(crate) mod rule_cache; pub(crate) mod rules_config; pub(crate) mod run_lint; +pub(crate) mod selection; pub(crate) mod skills; pub(crate) mod sql_test_objects; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index b57c779d18..1451384b04 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -4,13 +4,15 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::rules::{ builtin_catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, evaluation, lint, - lint_sql_bodies, lint_text, results, rule_cache, rules_config, run_lint, skills, + lint_expansion, lint_sql_bodies, lint_text, results, rule_cache, rules_config, run_lint, + selection, skills, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { evaluation::register(module)?; lint::register(module)?; lint_text::register(module)?; + lint_expansion::register(module)?; lint_sql_bodies::register(module)?; dsl_headers::register(module)?; custom_rule_evidence::register(module)?; @@ -19,6 +21,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { builtin_catalogue::register(module)?; rules_config::register(module)?; run_lint::register(module)?; + selection::register(module)?; results::register(module)?; skills::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs index 928f1410c1..ec95af0ea3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs @@ -14,6 +14,8 @@ const INCREMENTAL_STRATEGY_CONFIG: &str = "incremental_strategy"; const CURSOR_CONFIG: &str = "cursor"; const UNIQUE_KEY_CONFIG: &str = "unique_key"; const UNIQUE_AUDIT_COLUMN_ARGUMENT: &str = "column"; +const RELATION_KIND_SEED: &str = "seed"; +const RELATION_KIND_SOURCE: &str = "source"; /// `(kind, name, materialized, mode, strategy, cursor, unique key, columns, audits)`. pub(crate) type RelationKeyRow = ( @@ -143,14 +145,66 @@ fn declared_specs<'py>(entry: &Bound<'py, PyAny>, name: &str) -> PyResult, - rows: Vec, -) -> BTreeMap>> { +#[pyo3(signature = (*, model_rows, seeds, sources))] +fn declared_relation_keys<'py>( + py: Python<'py>, + model_rows: &Bound<'py, PyAny>, + seeds: &Bound<'py, PyAny>, + sources: &Bound<'py, PyAny>, +) -> PyResult> { + let mut rows: Vec = Vec::new(); + for row in model_rows.try_iter()? { + rows.push(row?.extract()?); + } + for (kind, entries) in [(RELATION_KIND_SEED, seeds), (RELATION_KIND_SOURCE, sources)] { + for entry in entries.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + let name: String = entry.getattr("name")?.extract()?; + rows.push(entry_relation_key_row( + kind, + name, + &PyDict::new(py), + &entry, + )?); + } + } let relations: Vec = rows.into_iter().map(declared_relation).collect(); - py.detach(|| derive_keys(&relations)) + let keys: BTreeMap>> = py.detach(|| derive_keys(&relations)); + let found: Bound<'py, PyDict> = PyDict::new(py); + for (identity, unique_keys) in keys { + let mut tuples: Vec> = Vec::with_capacity(unique_keys.len()); + for key in unique_keys { + tuples.push(PyTuple::new(py, key)?); + } + found.set_item(identity, PyTuple::new(py, tuples)?)?; + } + Ok(found) +} + +/// Declared keys of a compiled project's models, seeds and sources. +#[pyfunction] +fn compiled_relation_keys<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + let mut model_rows: Vec> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + model_rows.push(model?.getattr("relation_key_row")?); + } + let mut seeds: Vec> = Vec::new(); + for seed in project.getattr("seeds")?.try_iter()? { + seeds.push(seed?.getattr("schema_entry")?); + } + let mut sources: Vec> = Vec::new(); + for source in project.getattr("sources")?.try_iter()? { + sources.push(source?.getattr("source_entry")?); + } + declared_relation_keys( + py, + PyTuple::new(py, model_rows)?.as_any(), + PyTuple::new(py, seeds)?.as_any(), + PyTuple::new(py, sources)?.as_any(), + ) } fn declared_relation(row: RelationKeyRow) -> DeclaredRelation { @@ -177,5 +231,6 @@ fn declared_relation(row: RelationKeyRow) -> DeclaredRelation { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(entry_relation_key_row, module)?)?; - module.add_function(wrap_pyfunction!(declared_relation_keys, module)?) + module.add_function(wrap_pyfunction!(declared_relation_keys, module)?)?; + module.add_function(wrap_pyfunction!(compiled_relation_keys, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs index 0dfbbdf000..f2a30c19e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs @@ -14,8 +14,9 @@ use sqlbuild_core::format_command::main::collect_lint_files::collect_lint_files; use sqlbuild_rules::lint_text::main::first_header_only::first_header_only; use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, + KeywordArguments, call_with, imported, keywords, }; +use crate::bindings::_helpers::rules::lint_expansion::build_lint_expansion_context; const NATIVE_MODULE: &str = "sqlbuild._native"; const EXPANSION_MODULE: &str = "sqlbuild.lint._helpers.expansion"; @@ -167,13 +168,16 @@ fn expansion_context<'py>( if !run.config.getattr("native_enabled")?.is_truthy()? { return Ok(py.None().into_bound(py)); } - call_with( - &imported(py, EXPANSION_MODULE, "build_lint_expansion_context")?, - &[ - ("project_dir", run.project_dir.clone()), - ("value_renderer", arguments.value("value_renderer")?), - ("discovered_inputs", arguments.value("discovered_inputs")?), - ], + build_lint_expansion_context( + py, + Some(keywords( + py, + &[ + ("project_dir", run.project_dir.clone()), + ("value_renderer", arguments.value("value_renderer")?), + ("discovered_inputs", arguments.value("discovered_inputs")?), + ], + )?), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs new file mode 100644 index 0000000000..f0c25105c0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs @@ -0,0 +1,104 @@ +//! The active rule codes of a rules configuration and its custom rules, resolved natively. + +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_rules::rules::main::selected_codes::selected_codes_json; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; +const RULES_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const JSON_MODULE: &str = "json"; +const BUILTINS_MODULE: &str = "builtins"; + +/// The selected built-in and custom rule codes; invalid configuration raises `RulesError`. +#[pyfunction] +#[pyo3(signature = (*, config, custom_payloads))] +fn selected_rule_codes( + config: &Bound<'_, PyAny>, + custom_payloads: &Bound<'_, PyAny>, +) -> PyResult> { + let py: Python<'_> = config.py(); + let request: Bound<'_, PyDict> = PyDict::new(py); + request.set_item( + "version", + py.import(RULES_CONSTANTS_MODULE)? + .getattr("RULES_NATIVE_API_VERSION")?, + )?; + request.set_item( + "config", + py.import(DATACLASSES_MODULE)? + .getattr("asdict")? + .call1((config,))?, + )?; + request.set_item("custom_rules", custom_payloads)?; + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("sort_keys", true)?; + arguments.set_item("default", py.import(BUILTINS_MODULE)?.getattr("str")?)?; + let encoded: String = match py + .import(JSON_MODULE)? + .getattr("dumps")? + .call((request,), Some(&arguments)) + { + Ok(encoded) => encoded.extract()?, + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + let message: String = error.value(py).str()?.extract()?; + return Err(rules_error(py, &message, Some(error))); + } + Err(error) => return Err(error), + }; + let selected: Result = compiler_guard(|| Ok(selected_codes_json(&encoded)))?; + let codes: String = selected.map_err(|message| rules_error(py, &message, None))?; + serde_json::from_str::>(&codes).map_err(|_| { + rules_error( + py, + "native rules engine returned invalid selected rule codes", + None, + ) + }) +} + +/// Whether `code` is selected and not ignored by the configuration's prefixes. +#[pyfunction] +#[pyo3(signature = (*, config, code))] +fn rule_selected(config: &Bound<'_, PyAny>, code: &str) -> PyResult { + let matches = |field: &str| -> PyResult { + for selector in config.getattr(field)?.try_iter()? { + let selector: String = selector?.extract()?; + if code.starts_with(&selector) { + return Ok(true); + } + } + Ok(false) + }; + Ok(matches("select")? && !matches("ignore")?) +} + +/// `RulesError(message)`, caused by `cause` when given. +pub(crate) fn rules_error(py: Python<'_>, message: &str, cause: Option) -> PyErr { + let raised = || -> PyResult { + let error: Bound<'_, PyAny> = py + .import(RULES_EXCEPTIONS_MODULE)? + .getattr("RulesError")? + .call1((message,))?; + Ok(PyErr::from_value(error)) + }; + let error: PyErr = raised().unwrap_or_else(|error| error); + if let Some(cause) = cause { + error.set_cause(py, Some(cause)); + } + error +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(selected_rule_codes, module)?)?; + module.add_function(wrap_pyfunction!(rule_selected, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs index bb39d0f9c2..f6a103f885 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs @@ -1,28 +1,34 @@ //! Render and check the generated rules skill. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::rules::selection::rules_error; const SKILL_OWNER: &str = "sqlbuild"; const SKILL_IDENTITY: &str = "sqlbuild-rules"; +/// Generated guidance with its ownership header; a policy failure raises `RulesError`. #[pyfunction] -fn render_owned_skill(content: &str, input_fingerprint: &str) -> PyResult { - compiler_guard(|| { - fensu_policy::render_owned_skill( +#[pyo3(signature = (*, content, input_fingerprint))] +fn render_owned_skill(py: Python<'_>, content: &str, input_fingerprint: &str) -> PyResult { + let rendered: Result = compiler_guard(|| { + Ok(fensu_policy::render_owned_skill( SKILL_OWNER, SKILL_IDENTITY, input_fingerprint, content.as_bytes(), ) - .map_err(value_error) - .and_then(|value| String::from_utf8(value).map_err(value_error)) - }) + .map_err(|error| error.to_string()) + .and_then(|value| String::from_utf8(value).map_err(|error| error.to_string()))) + })?; + rendered.map_err(|message| rules_error(py, &message, None)) } +/// Whether generated guidance is current, stale or foreign under the ownership contract. #[pyfunction] +#[pyo3(signature = (*, content, input_fingerprint))] fn skill_freshness(content: Option<&str>, input_fingerprint: &str) -> String { let freshness = fensu_policy::skill_freshness( content.map(str::as_bytes), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs index 5c8be6460d..89fd26f640 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs @@ -12,6 +12,7 @@ use sqlbuild_core::scenario_command::main::render_snapshot_sync_result::render_s use sqlbuild_core::scenario_command::main::scenario_activity_message::scenario_activity_message; use sqlbuild_core::scenario_command::models::ScenarioNamespace; +use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::scenario_command::conversions::{ capture_row, scenario_result_row, write_local_runtime_target, write_warehouse_runtime_target, @@ -20,7 +21,6 @@ use crate::bindings::_helpers::scenario_command::setup::{ NATIVE_MODULE, boolean, flush, stream_isatty, text, write, }; -const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; /// How a finished scenario is reported. @@ -202,19 +202,12 @@ pub(crate) fn begin_section<'py>( (header, preamble, activity): (&str, &str, String), style: CliStyle, ) -> PyResult> { - let py: Python<'py> = stream.py(); write( stream, &format!("\n{}\n\n{preamble}", style.success_strong(header)), )?; flush(stream)?; - let status: Bound<'py, PyAny> = call_with( - &imported(py, STATUS_REPORTER_MODULE, "TransientStatusReporter")?, - &[ - ("stream", stream.clone()), - ("use_color", boolean(py, style.use_color)), - ], - )?; + let status: Bound<'py, PyAny> = status_reporter(stream, style.use_color)?; let status_is_tty: bool = stream_isatty(stream)?; if !status_is_tty { write(stream, &format!("{activity}\n\n"))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index f4fddaa6b2..cd6fd74b01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -26,9 +26,8 @@ use crate::bindings::_helpers::discovery::guidance::sql_analysis_guidance; pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; -const SCENARIO_CONFIG_MODULE: &str = - "sqlbuild.spec.contracts.main.resolve_effective_scenario_config"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; +const SCENARIO_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index 004178139b..05eefdb218 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -27,7 +27,7 @@ use crate::bindings::_helpers::scenario_command::setup::{ }; use crate::bindings::_helpers::scenario_command::snapshot_sync::{LocalSync, sync_local_snapshots}; -const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild.compiler.compile.main.effective_settings"; +const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild._native"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const SCENARIO_MODELS: &str = "sqlbuild.executor.scenario.models"; const LOCAL_ADAPTER: &str = "duckdb"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs index 33705c8763..ec680f1ab9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -87,7 +87,7 @@ fn diagnostic_row(diagnostic: InaccessibleDiagnostic) -> DiagnosticRow { /// `model_schema_scope_usages`: each schema's parent and enum `UsageRecord`s; S006s are reported. #[pyfunction] -fn model_schema_scope_usages<'py>( +pub(crate) fn model_schema_scope_usages<'py>( schema_files: &Bound<'py, PyAny>, resolver: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs index 8345733988..6c12986892 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs @@ -1,6 +1,9 @@ //! Expected-model relationship names and top-level CTEs of SQL tests and scenarios, scanned natively. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_scopes::relationship_names::main::expected_model_names::expected_model_names; use sqlbuild_scopes::relationship_names::main::top_level_ctes::scan_top_level_ctes; @@ -13,6 +16,9 @@ use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use crate::bindings::types::CompilerDetach; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; + /// One scan's error, or its values. type Scanned = (Option, Vec); @@ -72,7 +78,54 @@ fn scope_test_ctes( .collect()) } +/// One SQL test body's raw top-level CTEs; a scan failure raises `CompileInputError`. +#[pyfunction] +#[pyo3(signature = (*, sql, file_label, syntax))] +fn extract_sql_test_ctes<'py>( + py: Python<'py>, + sql: &str, + file_label: &str, + syntax: &Bound<'py, PyAny>, +) -> PyResult> { + let syntax: LexicalSyntax = syntax + .getattr("native_mapping")? + .extract::()? + .into(); + let outcome: TopLevelCtes = py + .compiler_detach(|| { + Ok(scan_top_level_ctes( + sql, + file_label, + RelationshipSource::Test, + &syntax, + )) + }) + .map_err(compiler_error)?; + let ctes: Vec<(String, String)> = match outcome { + TopLevelCtes::Scanned(ctes) => ctes, + TopLevelCtes::Failed(message) => { + let error: Bound<'py, PyAny> = py + .import(COMPILE_ERRORS_MODULE)? + .getattr("CompileInputError")? + .call1((message,))?; + return Err(PyErr::from_value(error)); + } + }; + let cte_class: Bound<'py, PyAny> = py + .import(COMPILE_MODELS_MODULE)? + .getattr("CompileSqlTestCte")?; + let mut built: Vec> = Vec::with_capacity(ctes.len()); + for (name, body) in ctes { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("name", name)?; + arguments.set_item("sql_body", body)?; + built.push(cte_class.call((), Some(&arguments))?); + } + PyTuple::new(py, built) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(extract_sql_test_ctes, module)?)?; module.add_function(wrap_pyfunction!(scope_expected_model_names, module)?)?; module.add_function(wrap_pyfunction!(scope_test_ctes, module)?)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs index bfef737d93..2087649059 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs @@ -5,7 +5,7 @@ use std::collections::BTreeSet; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_scopes::scope_report::models::ScopeResult; @@ -13,8 +13,13 @@ use sqlbuild_scopes::scope_text::main::render_scope_text::render_scope_text; use sqlbuild_scopes::scope_text::main::scope_result_failed::scope_result_failed; use sqlbuild_scopes::scope_text::models::ScopeTextRequest; +use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; use crate::bindings::_helpers::scopes::scope_objects::ScopeClasses; use crate::bindings::_helpers::scopes::scope_report_objects::read_result; +use crate::bindings::_helpers::scopes::scope_reports::{ + scope_browse, scope_explain, scope_list, scope_move_preview, scope_report, + serialize_scope_report, +}; const NATIVE_MODULE: &str = "sqlbuild._native"; const MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; @@ -218,40 +223,74 @@ fn merged<'py>( replaced(report, &changes) } +/// `value` as an optional Python string. +fn optional_text<'py>(py: Python<'py>, value: &Option) -> Option> { + value + .as_deref() + .map(|text| PyString::new(py, text).into_any()) +} + /// Run the query `request` selects over `lookup`. fn query_result<'py>( py: Python<'py>, lookup: &Bound<'py, PyAny>, (request, target): (&Bound<'py, PyAny>, Option), ) -> PyResult> { - let native: Bound<'_, PyModule> = py.import(NATIVE_MODULE)?; let at: Option = request.getattr("at")?.extract()?; let prospective: bool = at.is_some(); let browse: Option = request.getattr("browse")?.extract()?; if let Some(folder) = browse { - return native - .getattr("scope_browse")? - .call1((lookup, (folder, &target, prospective))); + return Ok(scope_browse( + py, + lookup, + (folder, optional_text(py, &target), prospective), + )? + .into_bound(py)); } let filters: Bound<'_, PyAny> = report_filters(py, request)?; let list_path: Option = request.getattr("list_path")?.extract()?; if let Some(folder) = list_path { - return native - .getattr("scope_list")? - .call1((lookup, (folder, &filters, &target, prospective))); + return Ok(scope_list( + py, + lookup, + ( + folder, + Some(filters), + optional_text(py, &target), + prospective, + ), + )? + .into_bound(py)); } let directory: bool = at .as_deref() .is_some_and(|at| at.ends_with('/') || at.ends_with('\\')); - let mut report: Bound<'_, PyAny> = native - .getattr("scope_report")? - .call1((lookup, (&target, &at, directory, &filters)))?; + let mut report: Bound<'_, PyAny> = scope_report( + py, + lookup, + ( + optional_text(py, &target), + optional_text(py, &at), + directory, + Some(filters), + ), + )? + .into_bound(py); let explain: Option = request.getattr("explain")?.extract()?; if let Some(declaration) = explain { - let (explanation, diagnostics): (Bound<'_, PyAny>, Bound<'_, PyAny>) = native - .getattr("scope_explain")? - .call1((lookup, (declaration, &target, &at, directory)))? - .extract()?; + let (explanation, diagnostics): (Bound<'_, PyAny>, Bound<'_, PyAny>) = scope_explain( + py, + lookup, + ( + PyString::new(py, &declaration).into_any(), + optional_text(py, &target), + optional_text(py, &at), + directory, + ), + ) + .map(|(explanation, diagnostics)| { + (explanation.into_bound(py), diagnostics.into_bound(py)) + })?; let complete: bool = explanation.getattr("complete")?.extract()?; report = merged( &report, @@ -261,10 +300,21 @@ fn query_result<'py>( } let as_path: Option = request.getattr("as_path")?.extract()?; if let Some(destination) = as_path { - let (preview, diagnostics): (Bound<'_, PyAny>, Bound<'_, PyAny>) = native - .getattr("scope_move_preview")? - .call1((lookup, target.clone().unwrap_or_default(), destination))? - .extract()?; + let (preview, diagnostics): (Bound<'_, PyAny>, Bound<'_, PyAny>) = scope_move_preview( + py, + lookup, + &PyString::new(py, &target.clone().unwrap_or_default()).into_any(), + &PyString::new(py, &destination).into_any(), + ) + .map(|(preview, diagnostics)| { + ( + preview.map_or_else( + || py.None().into_bound(py), + |preview| preview.into_bound(py), + ), + diagnostics.into_bound(py), + ) + })?; let complete: bool = !preview.is_none() && preview.getattr("complete")?.extract()?; report = merged(&report, ("move_preview", preview), (diagnostics, complete))?; } @@ -311,17 +361,14 @@ fn run_scope_command( ], )?; let index: Bound<'_, PyAny> = loader.call((), Some(&options))?; - let lookup: Bound<'_, PyAny> = native.getattr("build_scope_lookup")?.call1((index,))?; + let lookup: Bound<'_, PyAny> = build_scope_lookup(py, &index)?.into_bound(py); let target: Option = qualify_target(py, &lookup, request.getattr("target")?.extract()?)?; let result: Bound<'_, PyAny> = query_result(py, &lookup, (request, target.clone()))?; let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; let (output, native_result): (String, ScopeResult) = if request.getattr("json_output")?.extract()? { - let output: String = native - .getattr("serialize_scope_report")? - .call1((&result,))? - .extract()?; + let output: String = serialize_scope_report(py, &result)?; (output, read_result(&classes, &result)?) } else { let native_result: ScopeResult = read_result(&classes, &result)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs index 0aa58ddfae..d72e06f2ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs @@ -897,12 +897,18 @@ impl NativeScopeIndex { /// Python's `open_native_scope_index`: the natively built index of discovered inputs. #[pyfunction] -fn open_native_scope_index( +#[pyo3(signature = (*, discovered_inputs, loaded_macros, static_macros = None))] +pub(crate) fn open_native_scope_index( py: Python<'_>, discovered_inputs: &Bound<'_, PyAny>, loaded_macros: &Bound<'_, PyAny>, - static_macros: &Bound<'_, PyAny>, + static_macros: Option>, ) -> PyResult { + let static_macros: Bound<'_, PyAny> = match static_macros { + Some(static_macros) => static_macros, + None => PyTuple::empty(py).into_any(), + }; + let static_macros: &Bound<'_, PyAny> = &static_macros; let mut walk: Walk<'_> = Walk::new(py)?; walk.add_models(discovered_inputs)?; walk.add_resources(discovered_inputs)?; @@ -922,8 +928,30 @@ fn open_native_scope_index( materialise(walk, native) } +/// Python's `build_scope_index`: the natively built, partially valid index of discovered inputs. +#[pyfunction] +#[pyo3( + name = "build_scope_index", + signature = (*, discovered_inputs, loaded_macros = None, static_macros = None) +)] +fn build_scope_index_binding( + py: Python<'_>, + discovered_inputs: &Bound<'_, PyAny>, + loaded_macros: Option>, + static_macros: Option>, +) -> PyResult> { + let loaded_macros: Bound<'_, PyAny> = match loaded_macros { + Some(loaded_macros) if !loaded_macros.is_none() => loaded_macros, + _ => PyDict::new(py).into_any(), + }; + let native: NativeScopeIndex = + open_native_scope_index(py, discovered_inputs, &loaded_macros, static_macros)?; + Ok(native.index.clone_ref(py)) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_function(wrap_pyfunction!(open_native_scope_index, module)?)?; + module.add_function(wrap_pyfunction!(build_scope_index_binding, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index 516784814d..37bee60bec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -14,22 +14,21 @@ use crate::bindings::_helpers::discovery::scope_snapshot::discover_scope_snapsho use crate::bindings::_helpers::scopes::scope_cache::{ project_fingerprint, read_scope_index_cache, write_scope_index_cache, }; +use crate::bindings::_helpers::scopes::scope_index::open_native_scope_index; use crate::bindings::_helpers::scopes::scope_tolerant::scope_tolerant_index; +use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; const NATIVE_MODULE: &str = "sqlbuild._native"; const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; const SCOPE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.scopes.constants"; const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; -const COLLECTION_RENDERING_MODULE: &str = - "sqlbuild.spec.contracts.main.resolve_effective_collection_rendering"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; +const COLLECTION_RENDERING_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.adapter.discovery.main.resolve_adapter"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; -const COMPILE_USAGES_MODULE: &str = - "sqlbuild.compiler.compile.main._scope_index_with_compile_usages"; -const MACRO_INVENTORY_MODULE: &str = "sqlbuild.compiler.compile.main._static_macro_inventory"; +const MACRO_INVENTORY_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; /// Where the scope index cache file of a project lives: `(directory, file name)`. fn cache_location(project_dir: &Bound<'_, PyAny>) -> PyResult<(PathBuf, String)> { @@ -133,10 +132,7 @@ fn compiled_index<'py>( ("python_stage", python_stage), ], )?; - call_with( - &imported(py, COMPILE_USAGES_MODULE, "scope_index_with_compile_usages")?, - &[("inputs", compile_inputs)], - ) + scope_index_with_compile_usages(&compile_inputs) } /// The partial static index of tolerant discovery, with its faults as diagnostics. @@ -151,8 +147,16 @@ fn tolerant_index<'py>( &imported(py, MACRO_INVENTORY_MODULE, "inventory_project_macros")?, &[("macro_files", discovered.getattr("macro_files")?)], )?; - let native: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "open_native_scope_index")? - .call1((&discovered, PyDict::new(py), inventory.getattr("exports")?))?; + let native: Bound<'py, PyAny> = Bound::new( + py, + open_native_scope_index( + py, + &discovered, + &PyDict::new(py).into_any(), + Some(inventory.getattr("exports")?), + )?, + )? + .into_any(); let scan_store: Bound<'py, PyTuple> = PyTuple::new( py, [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs index f6dd0d5a72..a93b2a472f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs @@ -269,7 +269,8 @@ pub(crate) fn scope_path_error(py: Python<'_>, message: String) -> PyResult, index: &Bound<'_, PyAny>) -> PyResult> { +#[pyo3(signature = (*, index))] +pub(crate) fn build_scope_lookup(py: Python<'_>, index: &Bound<'_, PyAny>) -> PyResult> { let records: ReportRecords = report_records(&read_index(index)?); let built = py .compiler_detach(|| Ok(report_lookup(&records))) @@ -347,18 +348,17 @@ fn build_scope_lookup(py: Python<'_>, index: &Bound<'_, PyAny>) -> PyResult (Option, Option) { - match normalize_scope_path(path) { - Ok(normalized) => (Some(normalized), None), - Err(failure) => ( - None, - Some(path_message(&RejectedPath { - failure, - path: path.to_owned(), - })), - ), +#[pyo3(name = "normalize_scope_path", signature = (*, path))] +fn scope_normalize_path(py: Python<'_>, path: &Bound<'_, PyAny>) -> PyResult { + let path: String = path.str()?.extract()?; + match normalize_scope_path(&path) { + Ok(normalized) => Ok(normalized), + Err(failure) => Err(scope_path_error( + py, + path_message(&RejectedPath { failure, path }), + )?), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs index 211ba545f0..b4dc0d9265 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs @@ -4,6 +4,7 @@ use pyo3::exceptions::{PyIndexError, PyValueError}; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; +use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::json::models::JsonValue; use sqlbuild_scopes::scope_report::main::browse_scope_folders::browse_scope_folders; @@ -19,6 +20,7 @@ use sqlbuild_scopes::scope_report::models::{ ScopeResult, }; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::scopes::scope_lookup::{ NativeScopeLookup, identity_message, path_message, }; @@ -35,21 +37,21 @@ use crate::bindings::_helpers::scopes::scope_report_objects::{ type PyObject = Py; /// Target, location, whether it is a directory, and the filters. -type ReportArguments<'py> = ( +pub(crate) type ReportArguments<'py> = ( Option>, Option>, bool, Option>, ); /// Declaration, target, location and whether it is a directory. -type ExplainArguments<'py> = ( +pub(crate) type ExplainArguments<'py> = ( Bound<'py, PyAny>, Option>, Option>, bool, ); /// Folder, filters, target and whether the target is prospective. -type ListArguments<'py> = ( +pub(crate) type ListArguments<'py> = ( String, Option>, Option>, @@ -57,6 +59,7 @@ type ListArguments<'py> = ( ); const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const CURRENT_FOLDER: &str = "."; /// One lookup's native lookup and every declaration's values in input order. struct LookupReports { @@ -141,8 +144,7 @@ fn raised(diagnostics: Vec) -> Vec { } /// `query_scope_report`: one existing target or prospective path; invalid paths raise. -#[pyfunction] -fn scope_report( +pub(crate) fn scope_report( py: Python<'_>, lookup: &Bound<'_, PyAny>, request: ReportArguments<'_>, @@ -171,8 +173,7 @@ fn scope_report( } /// `explain_scope_declaration`: one qualified declaration in an optional context. -#[pyfunction] -fn scope_explain( +pub(crate) fn scope_explain( py: Python<'_>, lookup: &Bound<'_, PyAny>, request: ExplainArguments<'_>, @@ -218,8 +219,7 @@ fn folder_request( } /// `browse_scope_folders`: the direct child declaration-definition folders of `folder`. -#[pyfunction] -fn scope_browse( +pub(crate) fn scope_browse( py: Python<'_>, lookup: &Bound<'_, PyAny>, request: (String, Option>, bool), @@ -234,8 +234,7 @@ fn scope_browse( } /// `list_scope_declarations`: declarations recursively beneath one definition folder. -#[pyfunction] -fn scope_list( +pub(crate) fn scope_list( py: Python<'_>, lookup: &Bound<'_, PyAny>, request: ListArguments<'_>, @@ -257,7 +256,8 @@ fn scope_list( /// `preview_scope_move`: visibility deltas of a resource move, without mutating the lookup. #[pyfunction] -fn scope_move_preview( +#[pyo3(name = "preview_scope_move", signature = (*, lookup, resource, destination))] +pub(crate) fn scope_move_preview( py: Python<'_>, lookup: &Bound<'_, PyAny>, resource: &Bound<'_, PyAny>, @@ -295,7 +295,11 @@ fn serialized(classes: &ScopeClasses<'_>, result: &Bound<'_, PyAny>) -> PyResult /// `serialize_scope_report`: deterministic schema-version-two ASCII JSON of one result. #[pyfunction] -fn serialize_scope_report(py: Python<'_>, report: &Bound<'_, PyAny>) -> PyResult { +#[pyo3(signature = (*, report))] +pub(crate) fn serialize_scope_report( + py: Python<'_>, + report: &Bound<'_, PyAny>, +) -> PyResult { let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; match serialized(&classes, report) { Err(error) if error.is_instance_of::(py) => { @@ -310,18 +314,131 @@ fn serialize_scope_report(py: Python<'_>, report: &Bound<'_, PyAny>) -> PyResult /// `scope_metadata_projection` of one `ScopeIndex`, as the dictionary Python builds. #[pyfunction] -#[pyo3(name = "scope_metadata_projection")] +#[pyo3(name = "scope_metadata_projection", signature = (*, index))] fn metadata_projection(py: Python<'_>, index: &Bound<'_, PyAny>) -> PyResult { let index: ProjectionIndex = projection_index(read_index(index)?); let projection: JsonValue = scope_metadata_projection(&index); json_to_python(py, projection, false) } +fn optional<'py>( + arguments: &KeywordArguments<'py>, + name: &str, +) -> PyResult>> { + Ok(Some(arguments.value(name)?).filter(|value| !value.is_none())) +} + +/// Python's `query_scope_report`. +#[pyfunction] +#[pyo3(name = "query_scope_report", signature = (**options))] +fn query_scope_report_binding<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "query_scope_report", + options, + (&["lookup"], &["target", "at", "directory", "filters"]), + )?; + scope_report( + py, + &arguments.value("lookup")?, + ( + optional(&arguments, "target")?, + optional(&arguments, "at")?, + arguments.flag("directory")?, + optional(&arguments, "filters")?, + ), + ) +} + +/// Python's `explain_scope_declaration`. +#[pyfunction] +#[pyo3(name = "explain_scope_declaration", signature = (**options))] +fn explain_scope_declaration_binding<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult<(PyObject, PyObject)> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "explain_scope_declaration", + options, + (&["lookup", "declaration"], &["target", "at", "directory"]), + )?; + scope_explain( + py, + &arguments.value("lookup")?, + ( + arguments.value("declaration")?, + optional(&arguments, "target")?, + optional(&arguments, "at")?, + arguments.flag("directory")?, + ), + ) +} + +/// Python's `browse_scope_folders`. +#[pyfunction] +#[pyo3(name = "browse_scope_folders", signature = (**options))] +fn browse_scope_folders_binding<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "browse_scope_folders", + options, + (&["lookup"], &["folder", "target", "target_is_prospective"]), + )?; + let folder: String = match optional(&arguments, "folder")? { + Some(folder) => folder.extract()?, + None => CURRENT_FOLDER.to_owned(), + }; + scope_browse( + py, + &arguments.value("lookup")?, + ( + folder, + optional(&arguments, "target")?, + arguments.flag("target_is_prospective")?, + ), + ) +} + +/// Python's `list_scope_declarations`. +#[pyfunction] +#[pyo3(name = "list_scope_declarations", signature = (**options))] +fn list_scope_declarations_binding<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "list_scope_declarations", + options, + ( + &["lookup", "folder"], + &["filters", "target", "target_is_prospective"], + ), + )?; + scope_list( + py, + &arguments.value("lookup")?, + ( + arguments.value("folder")?.extract()?, + optional(&arguments, "filters")?, + optional(&arguments, "target")?, + arguments.flag("target_is_prospective")?, + ), + ) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scope_report, module)?)?; - module.add_function(wrap_pyfunction!(scope_explain, module)?)?; - module.add_function(wrap_pyfunction!(scope_browse, module)?)?; - module.add_function(wrap_pyfunction!(scope_list, module)?)?; + module.add_function(wrap_pyfunction!(query_scope_report_binding, module)?)?; + module.add_function(wrap_pyfunction!(explain_scope_declaration_binding, module)?)?; + module.add_function(wrap_pyfunction!(browse_scope_folders_binding, module)?)?; + module.add_function(wrap_pyfunction!(list_scope_declarations_binding, module)?)?; module.add_function(wrap_pyfunction!(scope_move_preview, module)?)?; module.add_function(wrap_pyfunction!(serialize_scope_report, module)?)?; module.add_function(wrap_pyfunction!(metadata_projection, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs index 2ffafd3ecf..5d969e577f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs @@ -1,6 +1,8 @@ //! Compile usage merging, placement validation and move relocation over a Python `ScopeIndex`. -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_scopes::placement::main::diagnostic_order::diagnostic_order; @@ -13,7 +15,7 @@ use sqlbuild_scopes::scope_index::models::ResourceIdentity; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::scopes::compile_usages::{ - MergeRows, SourceRow, UsageRow, merged_usage_rows, + MergeRows, SourceRow, UsageRow, merged_usage_rows, model_schema_scope_usages, }; use crate::bindings::_helpers::scopes::placement::{QualifiedRow, resource_identity}; use crate::bindings::_helpers::scopes::scope_objects::{ @@ -23,7 +25,17 @@ use crate::bindings::_helpers::scopes::scope_objects::{ use crate::bindings::types::CompilerDetach; const DATACLASSES_MODULE: &str = "dataclasses"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; const RUNTIME_USAGE: usize = 2; +/// Compile input groups whose items carry declaration usages, after the model inputs. +const COMPILE_USAGE_GROUPS: [&str; 5] = [ + "source_inputs", + "sql_function_inputs", + "audit_inputs", + "test_inputs", + "scenario_inputs", +]; /// `dataclasses.replace(item, **changes)`. pub(crate) fn replace<'py>( @@ -249,8 +261,11 @@ fn placement_validated_index<'py>( /// Complete usage and placement diagnostics, merging any collected compile usages first. #[pyfunction] -#[pyo3(signature = (index, enforce_placement, compile_usages))] -fn scope_placement_validated_index( +#[pyo3( + name = "get_placement_validated_scope_index", + signature = (*, index, enforce_placement = true, compile_usages = None) +)] +pub(crate) fn scope_placement_validated_index( index: &Bound<'_, PyAny>, enforce_placement: bool, compile_usages: Option>, @@ -264,7 +279,11 @@ fn scope_placement_validated_index( /// Declarations whose placement changes when `resource` moves, or `None` if unsettled. #[pyfunction] -fn scope_relocated_declarations( +#[pyo3( + name = "relocate_declarations_for_move", + signature = (*, index, resource, destination) +)] +pub(crate) fn scope_relocated_declarations( py: Python<'_>, index: &Bound<'_, PyAny>, resource: &Bound<'_, PyAny>, @@ -308,7 +327,79 @@ fn scope_relocated_declarations( Ok(Some(PyTuple::new(py, moved)?.unbind())) } +fn usage_records<'py>(records: &Bound<'py, PyAny>) -> PyResult>> { + records.try_iter()?.collect() +} + +/// Compile inputs' canonical index with all usage and placement facts. +pub(crate) fn scope_index_with_compile_usages<'py>( + inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let scope: Bound<'py, PyAny> = inputs.getattr("declaration_scope")?; + let resolver: Bound<'py, PyAny> = if scope.is_none() { + scope + } else { + scope.getattr("resolver")? + }; + let schema_files: Bound<'py, PyAny> = inputs + .getattr("discovered_inputs")? + .getattr("model_schema_files")?; + let mut usages: Vec> = Vec::new(); + usages.extend(usage_records( + model_schema_scope_usages(&schema_files, &resolver)?.as_any(), + )?); + for model in inputs.getattr("model_inputs")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + usages.extend(usage_records(&model.getattr("macro_usages")?)?); + usages.extend(usage_records(&model.getattr("declaration_usages")?)?); + } + for group in COMPILE_USAGE_GROUPS { + for item in inputs.getattr(group)?.try_iter()? { + usages.extend(usage_records(&item?.getattr("declaration_usages")?)?); + } + } + let enforce_placement: bool = inputs + .getattr("project_config")? + .getattr("scopes")? + .getattr("enforce_placement")? + .extract()?; + Ok(scope_placement_validated_index( + &inputs.getattr("scope_index")?, + enforce_placement, + Some(PyTuple::new(py, usages)?), + )? + .into_bound(py)) +} + +/// Python's `validate_scope_index`: raise one `ScopeValidationError` for its error diagnostics. +#[pyfunction] +#[pyo3(signature = (*, index))] +pub(crate) fn validate_scope_index(index: &Bound<'_, PyAny>) -> Result<(), PyErr> { + let py: Python<'_> = index.py(); + let error: Bound<'_, PyAny> = py + .import(SCOPE_TYPES_MODULE)? + .getattr("DiagnosticSeverity")? + .getattr("ERROR")?; + let mut errors: Vec> = Vec::new(); + for item in index.getattr("diagnostics")?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + if item.getattr("severity")?.is(&error) { + errors.push(item); + } + } + if errors.is_empty() { + return Ok(()); + } + Err(PyErr::from_value( + py.import(SCOPE_ERRORS_MODULE)? + .getattr("ScopeValidationError")? + .call1((PyTuple::new(py, errors)?,))?, + )) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(validate_scope_index, module)?)?; module.add_function(wrap_pyfunction!(scope_placement_validated_index, module)?)?; module.add_function(wrap_pyfunction!(scope_relocated_declarations, module)?)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs index 20f038e1f5..91425ea463 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs @@ -92,6 +92,7 @@ fn target_query<'py>( /// `query_target`: the `ScopeTargetQuery` of `target`; invalid text raises the scope error. #[pyfunction] +#[pyo3(name = "query_scope_target", signature = (*, lookup, target))] fn scope_query_target( py: Python<'_>, lookup: &Bound<'_, PyAny>, @@ -112,9 +113,30 @@ fn resolved( Ok(rows) } -/// Every visible fact of each matched resource, and its inaccessible declarations. +/// Python's `resolve_scope_visibility`: visible facts and inaccessible reasons per target. +#[pyfunction] +#[pyo3(signature = (*, lookup, target))] +fn resolve_scope_visibility( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + target: &Bound<'_, PyAny>, +) -> PyResult { + resolved_visibility(py, lookup, target, true) +} + +/// Python's `resolve_scope_declaration_visibility`: visible facts and inaccessible identities. #[pyfunction] -fn scope_resolve_visibility( +#[pyo3(signature = (*, lookup, target))] +fn resolve_scope_declaration_visibility( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + target: &Bound<'_, PyAny>, +) -> PyResult { + resolved_visibility(py, lookup, target, false) +} + +/// Every visible fact of each matched resource, and its inaccessible declarations. +fn resolved_visibility( py: Python<'_>, lookup: &Bound<'_, PyAny>, target: &Bound<'_, PyAny>, @@ -177,6 +199,7 @@ fn scope_resolve_visibility( /// Visible and inaccessible declarations for an authored path, including declaration files. #[pyfunction] +#[pyo3(name = "resolve_scope_path_visibility", signature = (*, lookup, path))] fn scope_resolve_path_visibility( py: Python<'_>, lookup: &Bound<'_, PyAny>, @@ -233,7 +256,11 @@ fn scope_resolve_path_visibility( pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(scope_query_target, module)?)?; - module.add_function(wrap_pyfunction!(scope_resolve_visibility, module)?)?; + module.add_function(wrap_pyfunction!(resolve_scope_visibility, module)?)?; + module.add_function(wrap_pyfunction!( + resolve_scope_declaration_visibility, + module + )?)?; module.add_function(wrap_pyfunction!(scope_resolve_path_visibility, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 61e4d53156..02231a9701 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -20,7 +20,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; -const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main._static_command"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const SEED_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._seed"; const SEED_STATE_MODULE: &str = "sqlbuild.compiler.planner.main.commands._seed_state"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs index f2eddbcf9e..273c987b1a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs @@ -1,17 +1,18 @@ //! Compile-owned inputs of the semantic completion stage: shapes, requests and SQL test columns. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; use crate::bindings::_helpers::analysis::binding_catalog::ignores_quoted_case; +use crate::bindings::_helpers::analysis::schema_request::{ + SchemaRequestParts, schema_validation_request, +}; +use crate::bindings::_helpers::semantic_checks::resource_sql::resource_sql_diagnostics_of; use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; +use crate::bindings::models::ProjectCatalog; const NATIVE_MODULE: &str = "sqlbuild._native"; const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; -const BINDING_REQUESTS_MODULE: &str = - "sqlbuild.compiler.compile._helpers.analysis.binding_requests"; -const RESOURCE_SQL_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.resource_sql"; -const DATACLASSES_MODULE: &str = "dataclasses"; /// The column names a metadata config value lists: a name, or the names of a list or tuple. pub(crate) fn column_names(value: &Bound<'_, PyAny>) -> PyResult> { @@ -106,30 +107,20 @@ pub(crate) fn revalidation_request<'py>( }; schema.set_item(name, shape)?; } - let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("query_sql", model.getattr("query_sql")?)?; - arguments.set_item("placeholders", py.None())?; - arguments.set_item("dialect", &dialect)?; - arguments.set_item( - "known_functions", - PyTuple::new(py, function_names(&functions)?)?, - )?; - arguments.set_item( - "known_types", - PyTuple::new(py, declared_types(&functions, shapes.as_any())?)?, - )?; - arguments.set_item("binding_schema", schema)?; - let request: Bound<'py, PyAny> = py - .import(BINDING_REQUESTS_MODULE)? - .getattr("get_complete_schema_binding_request")? - .call((), Some(&arguments))?; let ignores: bool = ignores_quoted_case(&project.getattr("effective_connection")?, &dialect)?; - let replaced: Bound<'py, PyDict> = PyDict::new(py); - replaced.set_item("catalog", project.getattr("binding_catalog")?)?; - replaced.set_item("quoted_identifiers_ignore_case", ignores)?; - py.import(DATACLASSES_MODULE)? - .getattr("replace")? - .call((request,), Some(&replaced)) + schema_validation_request( + py, + SchemaRequestParts { + query_sql: model.getattr("query_sql")?.extract()?, + placeholders: py.None().into_bound(py), + known_functions: PyTuple::new(py, function_names(&functions)?)?.into_any(), + known_types: PyTuple::new(py, declared_types(&functions, shapes.as_any())?)?.into_any(), + dialect, + schema: schema.into_any(), + catalog: project.getattr("binding_catalog")?, + quoted_identifiers_ignore_case: ignores, + }, + ) } /// The compile-time SQL checks of audits, SQL tests and SQL hooks. @@ -138,12 +129,12 @@ pub(crate) fn resource_sql_diagnostics<'py>( shapes: &Bound<'py, PyAny>, profile: &Bound<'py, PyAny>, ) -> PyResult> { - let py: Python<'py> = project.py(); - let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("project", project)?; - arguments.set_item("shapes", shapes)?; - arguments.set_item("profile", profile)?; - py.import(RESOURCE_SQL_MODULE)? - .getattr("get_resource_sql_diagnostics")? - .call((), Some(&arguments)) + let native: Bound<'py, PyAny> = project.getattr("binding_catalog")?.getattr("native")?; + Ok(resource_sql_diagnostics_of( + native.extract::>()?, + project, + shapes.cast::()?, + profile, + )? + .into_any()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index 5fb50715ed..d781c15a56 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -36,7 +36,7 @@ struct DiagnosticClasses<'py> { /// Bind the project's audit, SQL test and SQL hook bodies on `catalog` against `shapes`. #[pyfunction] -fn resource_sql_diagnostics_of<'py>( +pub(crate) fn resource_sql_diagnostics_of<'py>( catalog: PyRef<'_, ProjectCatalog>, project: &Bound<'py, PyAny>, shapes: &Bound<'py, PyDict>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs index eef36dc840..749b42b474 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs @@ -14,7 +14,7 @@ const OPEN_KINDS: [&str; 3] = [SOURCE_KIND, "ref", "dbt_ref"]; /// Partial-check reasons by model name; empty when SQL analysis is off. #[pyfunction] -fn semantic_coverage<'py>(project: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn semantic_coverage<'py>(project: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = project.py(); let reasons: Bound<'py, PyDict> = PyDict::new(py); if !project diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs index aef978d028..b705744b91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs @@ -9,6 +9,10 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::analysis::binding_catalog::{ self, CASE_SENSITIVE_BINDING_DIALECTS, NATIVE_DIALECT_ALIASES, schema_validations, }; +use crate::bindings::_helpers::analysis::schema_request::{ + SchemaRequestParts, schema_validation_request, +}; +use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; use crate::bindings::_helpers::semantic_shapes::shapes::{ @@ -21,10 +25,6 @@ const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const BINDING_UNKNOWN_TABLE_INTERNAL_CODE: &str = "B000"; -const BINDING_REQUESTS_MODULE: &str = - "sqlbuild.compiler.compile._helpers.analysis.binding_requests"; -const CURSOR_INTRINSICS_MODULE: &str = - "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; const DATACLASSES_MODULE: &str = "dataclasses"; const GRAPHLIB_MODULE: &str = "graphlib"; const REF_KIND: &str = "ref"; @@ -267,45 +267,30 @@ fn validated_diagnostics<'py>( let known_functions: Object<'py> = PyTuple::new(py, function_names(&functions)?)?.into_any(); let known_types: Object<'py> = PyTuple::new(py, declared_types(&functions, shapes.as_any())?)?.into_any(); - let request_builder: Object<'py> = py - .import(BINDING_REQUESTS_MODULE)? - .getattr("get_complete_schema_binding_request")?; - let cursor_sql: Object<'py> = py - .import(CURSOR_INTRINSICS_MODULE)? - .getattr("cursor_intrinsics_analysis_sql")?; - let ignores: Object<'py> = ignores_quoted_case(project, dialect)?; + let ignores: bool = + binding_catalog::ignores_quoted_case(&project.getattr("effective_connection")?, dialect)?; let mut requests: Vec> = Vec::with_capacity(models.len()); for model in models { - let query_sql: Object<'py> = call( - &cursor_sql, - &[ - ("sql", model.getattr("query_sql")?), - ( - "cursor_type", - model - .getattr("config")? - .getattr("values")? - .call_method1("get", ("cursor_type",))?, - ), - ], + let query_sql: String = analysis_cursor_sql( + py, + &model.getattr("query_sql")?.extract::()?, + &model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("cursor_type",))?, )?; - let request: Object<'py> = call( - &request_builder, - &[ - ("known_functions", known_functions.clone()), - ("known_types", known_types.clone()), - ("query_sql", query_sql), - ("placeholders", placeholders(model)?), - ("dialect", dialect.clone()), - ("binding_schema", model_inputs(model, shapes)?.into_any()), - ], - )?; - requests.push(replaced( - &request, - &[ - ("catalog", catalog.clone()), - ("quoted_identifiers_ignore_case", ignores.clone()), - ], + requests.push(schema_validation_request( + py, + SchemaRequestParts { + query_sql, + placeholders: placeholders(model)?, + dialect: dialect.clone(), + schema: model_inputs(model, shapes)?.into_any(), + known_functions: known_functions.clone(), + known_types: known_types.clone(), + catalog: catalog.clone(), + quoted_identifiers_ignore_case: ignores, + }, )?); } let results: Object<'py> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 1a0a4d7d90..c28e0f40d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -23,6 +23,7 @@ use sqlbuild_core::test_command::models::{ TestFailure, TestInspectionEntry, TestParameter, TestPlanWarning, }; +use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -32,17 +33,14 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; -const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main._static_command"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const TEST_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._sql_test"; const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; -const NESTED_PROGRESS_MODULE: &str = - "sqlbuild.cli.progress.classes.nested_command_progress_callbacks"; const PROGRESS_MODELS: &str = "sqlbuild.cli.progress.models"; -const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; const EXECUTOR_PIPELINE_MODELS: &str = "sqlbuild.executor.pipeline.models"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; -const RESOURCE_ID_MODULE: &str = "sqlbuild.executor.testing.main.resource_id"; +const RESOURCE_ID_MODULE: &str = "sqlbuild.executor.testing.main._resource_id"; const RUNTIME_TARGET_MODULE: &str = "sqlbuild.cli.target_artifacts.main._write_test_runtime_target"; const DECIMAL_KIND: &str = "decimal"; const PASS_OUTCOME: &str = "pass"; @@ -79,7 +77,7 @@ fn compact_json(payload: &Bound<'_, PyAny>) -> PyResult { } /// The label of one test or result: its name, plus source and values for parameterized cases. -fn test_label(item: &Bound<'_, PyAny>, name_field: &str) -> PyResult { +pub(crate) fn test_label(item: &Bound<'_, PyAny>, name_field: &str) -> PyResult { let name: String = item.getattr(name_field)?.extract()?; let source_path: Bound<'_, PyAny> = item.getattr("source_path")?; let values: Vec> = item @@ -486,7 +484,7 @@ fn run_tests<'py>( } let worker_count: i64 = concurrency.min(i64::try_from(test_count).unwrap_or(i64::MAX)); let progress: Bound<'py, PyAny> = call_with( - &imported(py, NESTED_PROGRESS_MODULE, "NestedCommandProgressCallbacks")?, + &imported(py, NATIVE_MODULE, "NestedCommandProgressCallbacks")?, &[ ("total", PyInt::new(py, test_count).into_any()), ("label", text(py, "test")), @@ -522,10 +520,7 @@ fn run_tests<'py>( ("use_color", use_color.clone()), ], )?; - let preflight: Bound<'py, PyAny> = call_with( - &imported(py, STATUS_REPORTER_MODULE, "TransientStatusReporter")?, - &[("stream", stream), ("use_color", use_color)], - )?; + let preflight: Bound<'py, PyAny> = status_reporter(&stream, use_color.is_truthy()?)?; let event_writer: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "ExecutionEventWriter")?.call0()?; let relay: Bound<'py, TestCallbackRelay> = Bound::new( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs index d516dcde60..f4eed0434b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs @@ -1,40 +1,16 @@ //! `@enum`/`@const` reference offsets scanned natively for the model loop. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_sqltext::compiler::main::declaration_references::scan_declaration_references; use sqlbuild_sqltext::compiler::models::{ DeclarationReference, DeclarationReferenceKind, DeclarationReferenceScan, DeclarationReferenceStop, }; -use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::types::CompilerDetach; - /// `(kind code, name, member, start, end)`: kind 0 is an enum member, 1 a constant. type ReferenceRow = (u8, String, Option, usize, usize); /// References and stop code: unclosed quote 0, block comment 1, invalid enum 2, constant 3. type ScanRow = (Vec, Option); -/// Return each SQL string's references and stopping error under this Python's `\\b`. -#[pyfunction] -fn scan_sql_declaration_references(py: Python<'_>, sqls: Vec) -> PyResult> { - scanned_rows(py, sqls, false) -} - -/// The same scan walking past malformed references, as Python's declaration usage walk did. -#[pyfunction] -fn scan_sql_declaration_usages(py: Python<'_>, sqls: Vec) -> PyResult> { - scanned_rows(py, sqls, true) -} - -fn scanned_rows(py: Python<'_>, sqls: Vec, skip_malformed: bool) -> PyResult> { - let scanned: Vec = py - .compiler_detach(|| Ok(scan_declaration_references(&sqls, skip_malformed))) - .map_err(compiler_error)?; - Ok(scanned.into_iter().map(scan_row).collect()) -} - /// One SQL string's references and stop code, scanned without walking past malformed ones. pub(crate) fn declaration_scan(sql: &str) -> ScanRow { let scanned: Vec = @@ -67,9 +43,3 @@ fn reference_row(reference: DeclarationReference) -> ReferenceRow { reference.end, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scan_sql_declaration_references, module)?)?; - module.add_function(wrap_pyfunction!(scan_sql_declaration_usages, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs index d94a7da68f..2b9fdf77b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs @@ -3,15 +3,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::sqltext::{ - code_scanner, declaration_references, model_headers, placeholders, sql_references, static_sql, + code_scanner, model_headers, placeholders, sql_references, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { model_headers::register(module)?; code_scanner::register(module)?; - static_sql::register(module)?; sql_references::register(module)?; - declaration_references::register(module)?; placeholders::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs index abb065732b..415807dfa7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs @@ -3,9 +3,9 @@ use std::cell::RefCell; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyString, PyStringMethods}; -use pyo3::{FromPyObject, PyErr, pyfunction, wrap_pyfunction}; +use pyo3::{FromPyObject, PyErr}; use sqlbuild_sqltext::compiler::main::sql_interpolation::interpolate_sql; use sqlbuild_sqltext::compiler::models::InterpolationRead; use sqlbuild_sqltext::compiler::types::InterpolationHost; @@ -142,15 +142,6 @@ fn key_text(key: &Bound<'_, PyAny>) -> String { key.str().map(|text| text.to_string()).unwrap_or_default() } -/// Interpolate each `(sql, file path)`; an error is returned in its row, not raised. -#[pyfunction] -fn interpolate_sql_batch<'py>( - sqls: Vec<(String, String)>, - sources: InterpolationSources<'py>, -) -> PyResult> { - interpolation_batch(sources, &sqls) -} - /// Each `(sql, file path)` interpolated in order; non-rendering Python failures raise. pub(crate) fn interpolation_batch( sources: InterpolationSources<'_>, @@ -211,8 +202,3 @@ fn read_rows(reads: Vec) -> Vec<(&'static str, String)> { }) .collect() } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(interpolate_sql_batch, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 3037c000d6..070cb9fca0 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -7,9 +7,9 @@ use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, contracts, cursor, dag_artifact, diagnostics, discovery, fingerprints, frontier, graph, - lineage, macro_bridge, manifest, model_config, plan_output, project_assembly, refactor, - references, render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, - state_rows, type_system, + lineage, macro_bridge, manifest, model_config, plan_output, planner_changes, planner_strategy, + project_assembly, refactor, references, render, rules, scopes, semantic_checks, + semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, }; use crate::bindings::models; @@ -45,6 +45,8 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { attachments::registration::register(module)?; refactor::registration::register(module)?; plan_output::registration::register(module)?; + planner_changes::registration::register(module)?; + planner_strategy::registration::register(module)?; references::registration::register(module)?; authored_values::registration::register(module)?; auditing::registration::register(module)?; diff --git a/fensu.toml b/fensu.toml index bec5e7a491..918a9a4fd8 100644 --- a/fensu.toml +++ b/fensu.toml @@ -28,11 +28,21 @@ rule = "FFR001" path = "src/sqlbuild/cli/commands/models.py" reason = "The TYPE_CHECKING block contains annotation-only imports, avoiding executor and integration imports for read-only commands; runtime declarations remain structured models." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/sql_values/constants.py" +reason = "Typed SQL values are normalized and state literals rendered natively; the namespace package keeps the value models, types, limits and errors the native bindings build and raise." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/cursor_algebra/__init__.py" reason = "The native cursor algebra in sqlbuild-cursor owns cursor behavior; this package only declares the cursor value models, sentinels and grain metadata callers import." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/spec/contracts/__init__.py" +reason = "Native bindings own configuration resolution; this package only declares the configuration models, constants and errors callers import." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/dag/types.py" @@ -83,11 +93,31 @@ rule = "FFR309" path = "src/sqlbuild/compiler/fingerprints/__init__.py" reason = "Fingerprint hashing and storage run natively; the package keeps the stored-row models, column and node-type vocabulary adapters and the executor share." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/compile/__init__.py" +reason = "Compilation runs natively; the package keeps the compile models, views, constants, errors, the diagnostic collector and the user-macro and Python-resource glue the native compiler calls back into." + +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/references/__init__.py" +reason = "SQL references are scanned natively; the package keeps the reference models, kinds and the reference-call patterns the Python planner still matches." + +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/resource_names/__init__.py" +reason = "Resource identities are validated natively; the package keeps the error class the native check raises." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/refactoring/__init__.py" reason = "Refactorings run natively; the package keeps the compiled-project input, manual-location model and error classes the native command reads and raises." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/scopes/__init__.py" +reason = "Declaration scopes are built, queried and validated natively; the package keeps the index, lookup and report models and the errors the native bindings raise." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/sql_test_glue/__init__.py" diff --git a/python-boundary.toml b/python-boundary.toml index e14c07366d..503fd8bbd2 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 62087 +remaining_lines = 51236 [[stays]] path = "executor" @@ -99,6 +99,16 @@ path = "diagnostics/classes/log_record_route_filter.py" reason = "public-api" note = "A logging.Filter that routes records to destinations." +[[stays]] +path = "diagnostics/main" +reason = "public-api" +note = "Diagnostics entry points built on Python logging: configuration, context, debug and SQL logging." + +[[stays]] +path = "diagnostics/_helpers/process_resources.py" +reason = "public-api" +note = "Samples process resources for debug logging through Python's resource module." + [[stays]] path = "rule_engine/_helpers/host" reason = "user-code" @@ -109,6 +119,11 @@ path = "rule_engine/_helpers/engine/custom_dispatch.py" reason = "user-code" note = "Submits custom rules to their worker thread beside native built-in rule evaluation." +[[stays]] +path = "rule_engine/_helpers/engine/custom_payloads.py" +reason = "user-code" +note = "Reads user rule functions (source lines, qualified names, import closures) to describe custom rules for native selection." + [[stays]] path = "rule_engine/_helpers/engine/custom_rules.py" reason = "user-code" @@ -124,6 +139,106 @@ path = "rule_engine/classes/rule_context.py" reason = "user-code" note = "The context object user rules receive." +[[stays]] +path = "rule_engine/_helpers/engine/definition.py" +reason = "user-code" +note = "The @rule decorator implementation; inspects user rule functions and their signatures." + +[[stays]] +path = "rule_engine/_helpers/engine/hermeticity.py" +reason = "user-code" +note = "Statically checks the imports of user rule source files." + +[[stays]] +path = "rule_engine/_helpers/engine/rule_harness.py" +reason = "user-code" +note = "Public custom-rule test harness that runs user rules over a real compiled project." + +[[stays]] +path = "rule_engine/_helpers/engine/fact_replay.py" +reason = "user-code" +note = "Recomputes the fact-view values a custom rule read, for its cache key." + +[[stays]] +path = "rule_engine/main/define.py" +reason = "user-code" +note = "Public @rule decorator entry point." + +[[stays]] +path = "rule_engine/main/evaluate_rule.py" +reason = "user-code" +note = "Public custom-rule test harness entry point." + +[[stays]] +path = "rule_engine/main/_evaluate.py" +reason = "user-code" +note = "Project evaluation entry used by the public custom-rule test harness." + +[[stays]] +path = "rule_engine/classes/audit_facts.py" +reason = "user-code" +note = "Fact view user rules read through RuleContext; the Python API over native data." + +[[stays]] +path = "rule_engine/classes/column_facts.py" +reason = "user-code" +note = "Fact view user rules read through RuleContext; the Python API over native data." + +[[stays]] +path = "rule_engine/classes/contract_facts.py" +reason = "user-code" +note = "Fact view user rules read through RuleContext; the Python API over native data." + +[[stays]] +path = "rule_engine/classes/declaration_facts.py" +reason = "user-code" +note = "Fact view user rules read through RuleContext; the Python API over native data." + +[[stays]] +path = "rule_engine/classes/graph_facts.py" +reason = "user-code" +note = "Fact view user rules read through RuleContext; the Python API over native data." + +[[stays]] +path = "rule_engine/classes/model_test_facts.py" +reason = "user-code" +note = "Fact view user rules read through RuleContext; the Python API over native data." + +[[stays]] +path = "rule_engine/classes/project_facts.py" +reason = "user-code" +note = "Fact view user rules read through RuleContext; the Python API over native data." + +[[stays]] +path = "rule_engine/classes/sql_facts.py" +reason = "user-code" +note = "Fact view user rules read through RuleContext; the Python API over native data." + +[[stays]] +path = "rule_engine/classes/project_tree.py" +reason = "user-code" +note = "Project-tree fact view user rules read through RuleContext." + +[[stays]] +path = "rule_engine/classes/sql_document.py" +reason = "user-code" +note = "Lazy Polyglot SQL document user rules read through RuleContext." + +[[stays]] +path = "rule_engine/classes/fact_reads.py" +reason = "user-code" +note = "Records the fact-view reads of one user rule evaluation." + +[[stays]] +path = "rule_engine/classes/fact_digests.py" +reason = "user-code" +note = "Digests the fact-view values user rules read, for the custom-rule cache." + +[[stays]] +path = "rule_engine/classes/shared_fact_encoder.py" +reason = "user-code" +note = "Encodes the Python fact-view values user rules read, for their digests." + [[stays]] path = "runtime/observability" reason = "public-api" @@ -194,6 +309,16 @@ path = "compiler/macro_bridge/_helpers/store_environment.py" reason = "user-code" note = "Fingerprints the Python interpreter and installed packages that project macros import from; the macro call store cache key." +[[stays]] +path = "rule_engine/classes/runtime_guard.py" +reason = "user-code" +note = "Guards custom-rule execution against side effects in user rule code." + +[[stays]] +path = "rule_engine/classes/module_state_monitor.py" +reason = "user-code" +note = "Watches user rule modules for state changes between custom-rule runs." + [[stays]] path = "compiler/compile/_helpers/render/user_macros.py" reason = "user-code" @@ -239,6 +364,96 @@ path = "compiler/pipeline/_helpers/materializations.py" reason = "user-code" note = "Imports user materialization modules." +[[stays]] +path = "compiler/frontier/_helpers/code_identity.py" +reason = "user-code" +note = "Fingerprints the loaded Python modules behind cached compiler and rule results." + +[[stays]] +path = "compiler/frontier/main/compiled_code_identity.py" +reason = "user-code" +note = "Entry point for the loaded-code identity of cached compiler and rule results." + +[[stays]] +path = "compiler/macro_bridge/main" +reason = "user-code" +note = "Macro bridge entry points for the running render stage that executes user macros." + +[[stays]] +path = "compiler/compile/_helpers/macro_bridge/call_store.py" +reason = "user-code" +note = "Hands the running macro bridge its user macro call store." + +[[stays]] +path = "compiler/python_nodes/main/identity.py" +reason = "user-code" +note = "Identity entry point over live user node functions." + +[[stays]] +path = "compiler/python_nodes/main/_hook_identities.py" +reason = "user-code" +note = "Builds identities of loaded user hook functions." + +[[stays]] +path = "compiler/python_nodes/main/_graph.py" +reason = "user-code" +note = "Builds the Python node graph from loaded user functions." + +[[stays]] +path = "compiler/python_nodes/main/_run_selection.py" +reason = "user-code" +note = "Builds the Python node graph from loaded user functions for run selection." + +[[stays]] +path = "compiler/auditing/main/above.py" +reason = "public-api" +note = "Audit bound helper project audit factories call." + +[[stays]] +path = "compiler/auditing/main/below.py" +reason = "public-api" +note = "Audit bound helper project audit factories call." + +[[stays]] +path = "compiler/auditing/main/outside.py" +reason = "public-api" +note = "Audit bound helper project audit factories call." + +[[stays]] +path = "compiler/auditing/main/audit_factory.py" +reason = "public-api" +note = "Decorator that declares project audit factories." + +[[stays]] +path = "compiler/auditing/main/get_audit_factory_definition.py" +reason = "public-api" +note = "Reads the declaration attached to a project audit factory." + +[[stays]] +path = "compiler/auditing/main/_builtins.py" +reason = "public-api" +note = "Built-in audit factories written against the public audit factory API." + +[[stays]] +path = "compiler/sql_analysis/main/import_polyglot.py" +reason = "public-api" +note = "Imports the Polyglot Python package for the Python API." + +[[stays]] +path = "compiler/sql_analysis/main/import_polyglot_sql.py" +reason = "public-api" +note = "Imports the Polyglot SQL Python package for the Python API." + +[[stays]] +path = "compiler/sql_analysis/_helpers/polyglot.py" +reason = "public-api" +note = "Wraps the Polyglot Python package with complexity guards." + +[[stays]] +path = "compiler/profiling" +reason = "public-api" +note = "Compile timing collectors built on Python context managers and clocks." + [[stays]] path = "assets.py" reason = "public-api" @@ -347,7 +562,3 @@ note = "A baseline that does not compile yields no relation keys, as before the [[handled_native_errors]] location = "lint/_helpers/fixes.py::_FixSession._verdict" note = "A candidate fix whose project fails to compile is unverified and not applied, as before the compile entry became native; no Python implementation runs instead." - -[[handled_native_errors]] -location = "compiler/project_assembly/_helpers/request.py::_opaque_render" -note = "A var that cannot render as text is recorded as the error a template using it raises; no Python implementation runs." diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index 11779304c8..0c241bde84 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -19,7 +19,6 @@ ) ALLOWED_MACRO_LOAD_PATHS: frozenset[str] = frozenset( { - "src/sqlbuild/compiler/compile/main/sql_expansion_context.py", "src/sqlbuild/compiler/compile/_helpers/render/user_macros.py", } ) diff --git a/scripts/planner_differential/README.md b/scripts/planner_differential/README.md new file mode 100644 index 0000000000..22586276f3 --- /dev/null +++ b/scripts/planner_differential/README.md @@ -0,0 +1,72 @@ +# Planner differential + +The compiler differential plans every project against an empty warehouse. The planner +differential checks the plans that depend on warehouse state: it builds each project, changes +it step by step, and records what the planner decides after every step. Use it around a change +that can alter planning (for example a port of planner code): record a baseline on the tree +before the change, then check the changed tree against it. + +```bash +# On the tree before the change (about 10 minutes with the defaults) +make planner-differential PLANNER_DIFFERENTIAL_MODE=record +# On the changed tree +make planner-differential +``` + +or call the script directly: + +```bash +uv run python -m scripts.run_planner_differential record --jobs 2 +uv run python -m scripts.run_planner_differential check --jobs 2 +uv run python -m scripts.run_planner_differential check --projects fixture/waffle_shop seed/0 +``` + +`check` exits 1 when any step output differs from the baseline or a project fails to run, and +prints a unified diff for every differing artifact. It writes the run it checked beside the +baseline (`.actual/`) for closer comparison. + +## Corpus + +The DuckDB fixture and example projects, then the first `--seeds` (default 6) generated seed +projects that are expected to compile and use the built-in DuckDB adapter. Generated seeds are +not required to build cleanly: model failures are part of the recorded outputs. `--projects` +keeps projects whose name contains any given text and `--limit` keeps the first N. + +## Mutation script + +`STEPS` in `constants.py` is the script every project runs, in order. Each `PlannerStep` names an +optional mutation, the kind of model it needs (`role`, with `index` choosing among the +candidates in name order), and extra flags for `sqb plan` and `sqb build`. Flags may contain +`{model}`, `{start_cursor}` and `{end_cursor}`, filled from the target model and its built +relation. A step whose role has no model, or whose placeholders have no value, is recorded as +skipped. Targets come from the previous unselected `sqb plan --json`, so later steps follow +earlier renames. + +The script covers a no-op rebuild, a query edit, a tag change (metadata only), a config change +(`on_schema_change`), a new declared column, a column rename and a model rename through +`sqb rename` (`migrate_from`), an added model, a relation dropped outside SQLBuild, source rows +added with a later cursor value, the `--max-microbatches`, cursor override, `--exclude` and +`--select` flags, a table-to-view materialization change, and `--full-refresh`. To +add a step, append a `PlannerStep`; a new kind of change adds a mutation function in +`_helpers/steps/mutations.py`. When a mutation makes planning fail, the step is recorded and the +project files are restored so later steps still run. `--steps` runs a subset (include `initial` +so later steps have targets). + +## What each step records + +- `sqb plan --json`: exit code, the document, and error lines; +- `sqb plan`: exit code and text; +- the planner's full `PlanOutput`, written as a canonical JSON capture when + `SQLBUILD_PLANNER_CAPTURE_DIR` names a directory (the same encoder as the compiler stage + captures); +- `sqb build`: exit code, summary line and error lines; +- the rows each `_sqlbuild*` state table gained and lost during the step. + +Run ids, invocation ids, event ids, timings, timestamps from the run date on (such as retention +expiry), and the run directories are masked. Mappings keyed by non-string values are sorted by +key. Baselines hold only masked values, so a baseline recorded earlier on the same day or on +another day compares cleanly unless planning changed. Plans that depend on the current time +beyond these masks would show up as differences; mask them in `_helpers/capturing/canonical.py`. + +Baselines are about 45 MB for the default corpus and live under +`/tmp/sqlbuild-planner-differential/` by default; they are not committed. diff --git a/scripts/planner_differential/__init__.py b/scripts/planner_differential/__init__.py new file mode 100644 index 0000000000..5263573d34 --- /dev/null +++ b/scripts/planner_differential/__init__.py @@ -0,0 +1 @@ +"""Stateful planner differential: plans and state across a fixed mutation script.""" diff --git a/src/sqlbuild/compiler/manifest/__init__.py b/scripts/planner_differential/_helpers/__init__.py similarity index 100% rename from src/sqlbuild/compiler/manifest/__init__.py rename to scripts/planner_differential/_helpers/__init__.py diff --git a/scripts/planner_differential/_helpers/baseline/__init__.py b/scripts/planner_differential/_helpers/baseline/__init__.py new file mode 100644 index 0000000000..8011b59e34 --- /dev/null +++ b/scripts/planner_differential/_helpers/baseline/__init__.py @@ -0,0 +1 @@ +"""Record and check planner baselines.""" diff --git a/scripts/planner_differential/_helpers/baseline/baseline.py b/scripts/planner_differential/_helpers/baseline/baseline.py new file mode 100644 index 0000000000..49556e77e7 --- /dev/null +++ b/scripts/planner_differential/_helpers/baseline/baseline.py @@ -0,0 +1,91 @@ +"""Write a run as the baseline, or compare it with the recorded baseline.""" + +from __future__ import annotations + +import difflib +import json +from pathlib import Path + +from scripts.compiler_differential._helpers.comparing.compare import as_json_object +from scripts.planner_differential.constants import BASELINE_SUFFIX, MAX_DIFF_LINES +from scripts.planner_differential.models import ProjectDifference + +_STEP_KEY: str = "step" +_STEPS_KEY: str = "steps" +_MISSING: str = "" + + +def write_document(*, directory: Path, slug: str, document: dict[str, object]) -> Path: + """Write one project's run as stable, indented JSON.""" + + directory.mkdir(parents=True, exist_ok=True) + path: Path = directory / f"{slug}{BASELINE_SUFFIX}" + _ = path.write_text(_rendered(document) + "\n", encoding="utf-8", newline="\n") + return path + + +def read_document(*, directory: Path, slug: str) -> dict[str, object] | None: + """Return a recorded project run, or None when the baseline has none.""" + + path: Path = directory / f"{slug}{BASELINE_SUFFIX}" + if not path.is_file(): + return None + return as_json_object(json.loads(path.read_text(encoding="utf-8"))) + + +def compare_documents( + *, expected: dict[str, object], actual: dict[str, object] +) -> tuple[ProjectDifference, ...]: + """Return a readable diff for every step artifact that differs from the baseline.""" + + expected_steps: list[dict[str, object]] = _steps(expected) + actual_steps: list[dict[str, object]] = _steps(actual) + differences: list[ProjectDifference] = [] + for index in range(max(len(expected_steps), len(actual_steps))): + left: dict[str, object] = expected_steps[index] if index < len(expected_steps) else {} + right: dict[str, object] = actual_steps[index] if index < len(actual_steps) else {} + step: str = str(right.get(_STEP_KEY, left.get(_STEP_KEY, index))) + for artifact in (*left, *(key for key in right if key not in left)): + if left.get(artifact, _MISSING) == right.get(artifact, _MISSING): + continue + differences.append( + ProjectDifference( + step=step, + artifact=artifact, + diff=unified_diff( + expected=left.get(artifact, _MISSING), + actual=right.get(artifact, _MISSING), + ), + ) + ) + return tuple(differences) + + +def unified_diff(*, expected: object, actual: object) -> str: + """Return a bounded unified diff of two JSON values rendered one item per line.""" + + lines: list[str] = list( + difflib.unified_diff( + _rendered(expected).splitlines(), + _rendered(actual).splitlines(), + fromfile="baseline", + tofile="actual", + lineterm="", + n=2, + ) + ) + if len(lines) > MAX_DIFF_LINES: + omitted: int = len(lines) - MAX_DIFF_LINES + lines = [*lines[:MAX_DIFF_LINES], f"... {omitted} more diff lines"] + return "\n".join(lines) + + +def _steps(document: dict[str, object]) -> list[dict[str, object]]: + steps: object = document.get(_STEPS_KEY) + if not isinstance(steps, list): + return [] + return [mapping for step in steps if (mapping := as_json_object(step)) is not None] + + +def _rendered(value: object) -> str: + return json.dumps(value, indent=1, ensure_ascii=False) diff --git a/scripts/planner_differential/_helpers/capturing/__init__.py b/scripts/planner_differential/_helpers/capturing/__init__.py new file mode 100644 index 0000000000..ebf1112e2a --- /dev/null +++ b/scripts/planner_differential/_helpers/capturing/__init__.py @@ -0,0 +1 @@ +"""Capture and canonicalise plans and state rows.""" diff --git a/scripts/planner_differential/_helpers/capturing/canonical.py b/scripts/planner_differential/_helpers/capturing/canonical.py new file mode 100644 index 0000000000..574825e90f --- /dev/null +++ b/scripts/planner_differential/_helpers/capturing/canonical.py @@ -0,0 +1,95 @@ +"""Mask run-specific values so two runs of the same tree compare equal.""" + +from __future__ import annotations + +import json +import re +from datetime import date, timedelta + +from scripts.compiler_differential.classes.capture_file import expand_capture_text +from scripts.planner_differential.constants import ( + COMPACT_TIMESTAMP_PATTERN, + ELAPSED_TIME_MASK, + ELAPSED_TIME_PATTERN, + ISO_TIMESTAMP_PATTERN, + MAPPING_MARKER, + RECENT_TIMESTAMP_DAYS, + RECENT_TIMESTAMP_MASK, + VOLATILE_KEY_SUFFIXES, + VOLATILE_KEYS, + VOLATILE_VALUE_MASK, +) +from scripts.planner_differential.models import TextMasks +from sqlbuild._native import mask_capture_noise + + +def mask_text(*, text: str, masks: TextMasks) -> str: + """Mask invocation ids, run paths and timestamps near the run date.""" + + masked: str = mask_capture_noise(text) + for prefix, replacement in masks.replacements: + masked = masked.replace(prefix, replacement) + masked = ISO_TIMESTAMP_PATTERN.sub( + lambda match: _recent_or_kept(match=match, today=masks.today, iso=True), masked + ) + return COMPACT_TIMESTAMP_PATTERN.sub( + lambda match: _recent_or_kept(match=match, today=masks.today, iso=False), masked + ) + + +def mask_console_text(*, text: str, masks: TextMasks) -> str: + """Mask console output: the artifact noise plus elapsed times.""" + + return ELAPSED_TIME_PATTERN.sub(ELAPSED_TIME_MASK, mask_text(text=text, masks=masks)) + + +def canonical_value(*, value: object, masks: TextMasks) -> object: + """Return decoded JSON with volatile keys and run-specific text masked.""" + + if isinstance(value, dict): + canonical: dict[str, object] = { + str(key): ( + VOLATILE_VALUE_MASK + if item is not None and _volatile_key(str(key)) + else canonical_value(value=item, masks=masks) + ) + for key, item in value.items() + } + pairs: object = canonical.get(MAPPING_MARKER) + if len(canonical) == 1 and isinstance(pairs, list): + return {MAPPING_MARKER: sorted(pairs, key=_sort_key)} + return canonical + if isinstance(value, list): + return [canonical_value(value=item, masks=masks) for item in value] + if isinstance(value, str): + return mask_text(text=value, masks=masks) + return value + + +def canonical_plan_capture(*, text: str, masks: TextMasks) -> object: + """Expand a plan capture document's shared nodes and mask its volatile values.""" + + return canonical_value(value=expand_capture_text(text), masks=masks) + + +def _volatile_key(key: str) -> bool: + return key in VOLATILE_KEYS or key.endswith(VOLATILE_KEY_SUFFIXES) + + +def _sort_key(pair: object) -> str: + key: object = pair[0] if isinstance(pair, list) and pair else pair + return json.dumps(key, sort_keys=True, ensure_ascii=False) + + +def _recent_or_kept(*, match: re.Match[str], today: date, iso: bool) -> str: + day_text: str = match.group(1) + try: + day: date = ( + date.fromisoformat(day_text) + if iso + else date(int(day_text[:4]), int(day_text[4:6]), int(day_text[6:])) + ) + except ValueError: + return match.group(0) + recent: bool = day >= today - timedelta(days=RECENT_TIMESTAMP_DAYS) + return RECENT_TIMESTAMP_MASK if recent else match.group(0) diff --git a/scripts/planner_differential/_helpers/capturing/state_tables.py b/scripts/planner_differential/_helpers/capturing/state_tables.py new file mode 100644 index 0000000000..d29fc7c4d9 --- /dev/null +++ b/scripts/planner_differential/_helpers/capturing/state_tables.py @@ -0,0 +1,117 @@ +"""Read SQLBuild state-table rows from a project's DuckDB files.""" + +from __future__ import annotations + +import base64 +import binascii +import json +from collections import Counter +from datetime import date, datetime, time +from decimal import Decimal +from pathlib import Path + +import duckdb + +from scripts.planner_differential._helpers.capturing.canonical import canonical_value +from scripts.planner_differential.constants import ( + BASE64_COLUMN_SUFFIX, + DUCKDB_SUFFIX, + STATE_TABLE_PREFIX, +) +from scripts.planner_differential.models import TextMasks + +_STATE_TABLES_SQL: str = ( + "SELECT table_schema, table_name FROM information_schema.tables " + "WHERE table_type = 'BASE TABLE' AND starts_with(table_name, ?) " + "ORDER BY table_schema, table_name" +) + + +def database_files(project_dir: Path) -> tuple[Path, ...]: + """Return the project's DuckDB files in a stable order.""" + + return tuple(sorted(project_dir.rglob(f"*{DUCKDB_SUFFIX}"))) + + +def state_rows(*, project_dir: Path, masks: TextMasks) -> dict[str, list[str]]: + """Return each state table's masked rows as compact JSON, sorted.""" + + tables: dict[str, list[str]] = {} + for path in database_files(project_dir): + connection: duckdb.DuckDBPyConnection = duckdb.connect(str(path), read_only=True) + try: + names: list[tuple[str, str]] = connection.execute( + _STATE_TABLES_SQL, [STATE_TABLE_PREFIX] + ).fetchall() + for schema, table in names: + key: str = f"{path.relative_to(project_dir).as_posix()}:{schema}.{table}" + tables[key] = _table_rows( + connection=connection, relation=f'"{schema}"."{table}"', masks=masks + ) + finally: + connection.close() + return tables + + +def state_delta( + *, before: dict[str, list[str]], after: dict[str, list[str]] +) -> dict[str, dict[str, list[object]]]: + """Return the rows each state table gained and lost, as decoded JSON.""" + + delta: dict[str, dict[str, list[object]]] = {} + for table in sorted({*before, *after}): + old: Counter[str] = Counter(before.get(table, [])) + new: Counter[str] = Counter(after.get(table, [])) + added: list[str] = sorted((new - old).elements()) + removed: list[str] = sorted((old - new).elements()) + if added or removed: + delta[table] = { + "added": [json.loads(row) for row in added], + "removed": [json.loads(row) for row in removed], + } + return delta + + +def _table_rows( + *, connection: duckdb.DuckDBPyConnection, relation: str, masks: TextMasks +) -> list[str]: + cursor: duckdb.DuckDBPyConnection = connection.execute(f"SELECT * FROM {relation}") + columns: list[str] = [str(column[0]) for column in cursor.description or ()] + rows: list[str] = [] + for row in cursor.fetchall(): + decoded: dict[str, object] = { + column: _decoded(column=column, value=value) + for column, value in zip(columns, row, strict=True) + } + rows.append( + json.dumps( + canonical_value(value=decoded, masks=masks), + sort_keys=True, + ensure_ascii=False, + default=str, + ) + ) + return sorted(rows) + + +def _decoded(*, column: str, value: object) -> object: + if isinstance(value, datetime | date | time): + return value.isoformat() + if isinstance(value, Decimal): + return str(value) + if isinstance(value, bytes): + return value.hex() + if isinstance(value, str) and column.endswith(BASE64_COLUMN_SUFFIX): + return _base64_json(value) + return value + + +def _base64_json(value: str) -> object: + try: + text: str = base64.b64decode(value, validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError): + return value + try: + return json.loads(text) + except json.JSONDecodeError: + return text diff --git a/scripts/planner_differential/_helpers/corpus/__init__.py b/scripts/planner_differential/_helpers/corpus/__init__.py new file mode 100644 index 0000000000..3c565581d0 --- /dev/null +++ b/scripts/planner_differential/_helpers/corpus/__init__.py @@ -0,0 +1 @@ +"""Select the projects the planner differential runs.""" diff --git a/scripts/planner_differential/_helpers/corpus/corpus.py b/scripts/planner_differential/_helpers/corpus/corpus.py new file mode 100644 index 0000000000..2d432ae711 --- /dev/null +++ b/scripts/planner_differential/_helpers/corpus/corpus.py @@ -0,0 +1,75 @@ +"""Select the fixture, example and generated projects the planner differential runs.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +from scripts.compiler_differential.classes.project_builder import ProjectBuilder +from scripts.compiler_differential.models import GeneratedProject +from scripts.planner_differential.constants import ( + DUCKDB_ADAPTER, + EXAMPLE_ROOT, + FIXTURE_ROOT, + PROJECT_CONFIG_FILE, +) +from scripts.planner_differential.models import PlannerProject + +_SEED_SEARCH_FACTOR: int = 10 +_FIXTURE_KIND: str = "fixture" +_EXAMPLE_KIND: str = "example" + + +def build_corpus(*, repo_root: Path, seed_start: int, seeds: int) -> list[PlannerProject]: + """Return DuckDB fixtures and examples, then the first `seeds` valid built-in DuckDB seeds.""" + + projects: list[PlannerProject] = [] + for kind, root in ((_FIXTURE_KIND, FIXTURE_ROOT), (_EXAMPLE_KIND, EXAMPLE_ROOT)): + projects.extend( + PlannerProject(name=f"{kind}/{path.name}", source_dir=path) + for path in _duckdb_projects(repo_root / root) + ) + found: int = 0 + for seed in range(seed_start, seed_start + seeds * _SEED_SEARCH_FACTOR): + if found == seeds: + break + generated: GeneratedProject = ProjectBuilder(seed).build() + if generated.expected_error_code is not None or not _duckdb_config( + generated.files.get(PROJECT_CONFIG_FILE, "") + ): + continue + projects.append(PlannerProject(name=f"seed/{seed}", writer=generated.write)) + found += 1 + return projects + + +def filter_corpus( + *, projects: list[PlannerProject], names: tuple[str, ...], limit: int | None +) -> list[PlannerProject]: + """Keep projects whose name contains any filter, then the first `limit` of them.""" + + kept: list[PlannerProject] = [ + project for project in projects if not names or _matches(name=project.name, filters=names) + ] + return kept if limit is None else kept[:limit] + + +def _matches(*, name: str, filters: tuple[str, ...]) -> bool: + return any(item in name for item in filters) + + +def _duckdb_projects(root: Path) -> list[Path]: + return sorted( + path + for path in root.iterdir() + if path.is_dir() + and (path / PROJECT_CONFIG_FILE).is_file() + and _duckdb_config((path / PROJECT_CONFIG_FILE).read_text(encoding="utf-8")) + ) + + +def _duckdb_config(text: str) -> bool: + try: + return tomllib.loads(text).get("adapter") == DUCKDB_ADAPTER + except tomllib.TOMLDecodeError: + return False diff --git a/scripts/planner_differential/_helpers/running/__init__.py b/scripts/planner_differential/_helpers/running/__init__.py new file mode 100644 index 0000000000..5d7f937507 --- /dev/null +++ b/scripts/planner_differential/_helpers/running/__init__.py @@ -0,0 +1 @@ +"""Run one project through the mutation script.""" diff --git a/scripts/planner_differential/_helpers/running/commands.py b/scripts/planner_differential/_helpers/running/commands.py new file mode 100644 index 0000000000..44e315da88 --- /dev/null +++ b/scripts/planner_differential/_helpers/running/commands.py @@ -0,0 +1,56 @@ +"""Run sqb in a fresh process with the harness environment.""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +from scripts.compiler_differential.constants import GENERATOR_ENVIRONMENT +from scripts.planner_differential.constants import ( + COMMAND_TIMEOUT_SECONDS, + EXCLUDED_ENVIRONMENT_KEYS, + EXCLUDED_ENVIRONMENT_PREFIX, + SQB_ENTRY, +) +from scripts.planner_differential.models import CommandResult + +_TIMEOUT_EXIT_CODE: int = -1 + + +def harness_environment() -> dict[str, str]: + """Return this environment without capture or dbt settings, plus generator variables.""" + + return { + **{ + key: value + for key, value in os.environ.items() + if key not in EXCLUDED_ENVIRONMENT_KEYS + and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX) + }, + **GENERATOR_ENVIRONMENT, + } + + +def run_sqb( + *, arguments: tuple[str, ...], project_dir: Path, environment: dict[str, str], python: Path +) -> CommandResult: + """Run one sqb command in `project_dir`; a timeout reports exit code -1.""" + + try: + completed: subprocess.CompletedProcess[str] = subprocess.run( + [str(python), "-c", SQB_ENTRY, "--no-color", *arguments], + cwd=project_dir, + env=environment, + capture_output=True, + text=True, + encoding="utf-8", + errors="surrogateescape", + timeout=COMMAND_TIMEOUT_SECONDS, + check=False, + ) + except subprocess.TimeoutExpired: + return CommandResult(exit_code=_TIMEOUT_EXIT_CODE, stdout="", stderr="timed out") + return CommandResult( + exit_code=completed.returncode, stdout=completed.stdout, stderr=completed.stderr + ) diff --git a/scripts/planner_differential/_helpers/running/options.py b/scripts/planner_differential/_helpers/running/options.py new file mode 100644 index 0000000000..d09e560cd2 --- /dev/null +++ b/scripts/planner_differential/_helpers/running/options.py @@ -0,0 +1,79 @@ +"""Parse the planner differential command line into run options.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from scripts.planner_differential.constants import ( + ACTUAL_SUFFIX, + DEFAULT_BASELINE_DIR, + DEFAULT_JOBS, + DEFAULT_SEED_COUNT, + MODE_CHECK, + MODES, + STEPS, +) +from scripts.planner_differential.models import RunOptions + + +def parse_args(argv: list[str] | None) -> argparse.Namespace: + """Parse and validate the harness arguments.""" + + parser: argparse.ArgumentParser = argparse.ArgumentParser( + prog="python -m scripts.run_planner_differential", + description=( + "Build each corpus project, apply a fixed mutation script, and record or check " + "plans, plan outputs and state-table rows after every step." + ), + ) + parser.add_argument("mode", choices=MODES, help="record a baseline, or check against it") + parser.add_argument( + "--baseline-dir", + type=Path, + default=Path(DEFAULT_BASELINE_DIR), + help=( + f"baseline directory (default: {DEFAULT_BASELINE_DIR}); check writes the run " + f"beside it with an {ACTUAL_SUFFIX} suffix" + ), + ) + parser.add_argument("--jobs", type=int, default=DEFAULT_JOBS) + parser.add_argument("--seeds", type=int, default=DEFAULT_SEED_COUNT) + parser.add_argument("--seed-start", type=int, default=0) + parser.add_argument( + "--projects", + nargs="*", + default=[], + help="keep projects whose name contains any of these, such as fixture/ or seed/3", + ) + parser.add_argument( + "--steps", + nargs="*", + choices=[step.name for step in STEPS], + default=[], + help="run only these steps of the mutation script, in script order", + ) + parser.add_argument("--limit", type=int, default=None, help="run only the first N projects") + parser.add_argument( + "--work-dir", type=Path, default=None, help="keep project copies and captures here" + ) + parser.add_argument("--python", type=Path, default=Path(sys.executable)) + args: argparse.Namespace = parser.parse_args(argv) + if args.mode == MODE_CHECK and not args.baseline_dir.is_dir(): + parser.error(f"no baseline at {args.baseline_dir}; run with `record` first") + return args + + +def run_options(*, args: argparse.Namespace, work_dir: Path) -> RunOptions: + """Return run options from parsed arguments, running in `work_dir`.""" + + return RunOptions( + work_dir=work_dir.absolute(), + jobs=args.jobs, + python=args.python, + mode=args.mode, + baseline_dir=args.baseline_dir.absolute(), + keep_work=args.work_dir is not None, + steps=tuple(step for step in STEPS if not args.steps or step.name in args.steps), + ) diff --git a/scripts/planner_differential/_helpers/running/project_run.py b/scripts/planner_differential/_helpers/running/project_run.py new file mode 100644 index 0000000000..0a39cd4c67 --- /dev/null +++ b/scripts/planner_differential/_helpers/running/project_run.py @@ -0,0 +1,227 @@ +"""Run one project through the mutation script and record every step's outputs.""" + +from __future__ import annotations + +import json +import shutil +from datetime import date +from pathlib import Path + +from scripts.compiler_differential._helpers.comparing.compare import as_json_object +from scripts.planner_differential._helpers.capturing.canonical import ( + canonical_plan_capture, + canonical_value, + mask_console_text, + mask_text, +) +from scripts.planner_differential._helpers.capturing.state_tables import ( + state_delta, + state_rows, +) +from scripts.planner_differential._helpers.running.commands import harness_environment, run_sqb +from scripts.planner_differential._helpers.running.snapshot import ( + project_files, + restore_project_files, +) +from scripts.planner_differential._helpers.steps.arguments import step_arguments +from scripts.planner_differential._helpers.steps.mutations import apply_mutation +from scripts.planner_differential._helpers.steps.targets import plan_models, role_target +from scripts.planner_differential.constants import ( + BUILD_SUMMARY_PREFIXES, + CAPTURE_DIRECTORY, + COPY_IGNORED, + ERROR_LINE_PATTERN, + PLAN_CAPTURE_ENV_VAR, + PROJECT_DIRECTORY, + PROJECT_PATH_MASK, + WORK_PATH_MASK, +) +from scripts.planner_differential.models import ( + CommandResult, + ModelFacts, + MutationResult, + PlannerProject, + RunOptions, + StepContext, + TextMasks, +) + +_SOURCE_MAP_KEY: str = "source_map" +_SELECTION_FLAGS: frozenset[str] = frozenset({"--select", "--exclude"}) + + +def project_slug(name: str) -> str: + """Return a file name for a project name such as `fixture/orders`.""" + + return name.replace("/", "__") + + +def run_project(*, project: PlannerProject, options: RunOptions) -> dict[str, object]: + """Build the project once per step after its mutation and return the step records.""" + + case_dir: Path = options.work_dir / project_slug(project.name) + project_dir: Path = _prepared_project(project=project, case_dir=case_dir) + masks: TextMasks = TextMasks( + replacements=tuple( + (str(path), mask) + for path, mask in ( + (project_dir.resolve(), PROJECT_PATH_MASK), + (project_dir, PROJECT_PATH_MASK), + (options.work_dir.resolve(), WORK_PATH_MASK), + (options.work_dir, WORK_PATH_MASK), + ) + ), + today=date.today(), + ) + environment: dict[str, str] = harness_environment() + models: tuple[ModelFacts, ...] = () + source_tables: tuple[str, ...] = () + previous_state: dict[str, list[str]] = {} + records: list[dict[str, object]] = [] + for index, step in enumerate(options.steps): + context: StepContext = StepContext( + project_dir=project_dir, + models=models, + source_tables=source_tables, + environment=environment, + python=options.python, + ) + record: dict[str, object] = {"step": step.name} + records.append(record) + target: ModelFacts | None = ( + None + if step.role is None + else role_target( + role=step.role, index=step.index, models=models, project_dir=project_dir + ) + ) + if step.role is not None and target is None: + record["skipped"] = f"no {step.role} model" + continue + arguments: tuple[str, ...] | None = step_arguments( + step=step, context=context, target=target + ) + if arguments is None: + record["skipped"] = "no value for a flag placeholder" + continue + saved: dict[str, bytes] = project_files(project_dir) + if step.mutation is not None and target is not None: + mutation: MutationResult = apply_mutation( + name=step.mutation, context=context, target=target + ) + record["mutation"] = { + "applied": mutation.applied, + "note": mask_text(text=mutation.note, masks=masks), + } + record["arguments"] = list(arguments) + capture_dir: Path = case_dir / CAPTURE_DIRECTORY / f"{index:02d}-{step.name}" + shutil.rmtree(capture_dir, ignore_errors=True) + plan_json: CommandResult = run_sqb( + arguments=("plan", "--json", *arguments), + project_dir=project_dir, + environment={**environment, PLAN_CAPTURE_ENV_VAR: str(capture_dir)}, + python=options.python, + ) + record["plan_json"] = _plan_json_record(result=plan_json, masks=masks) + plan_output: list[object] = _plan_capture(capture_dir=capture_dir, masks=masks) + record["plan_output"] = plan_output + plan_text: CommandResult = run_sqb( + arguments=("plan", *arguments), + project_dir=project_dir, + environment=environment, + python=options.python, + ) + record["plan_text"] = { + "exit_code": plan_text.exit_code, + "lines": mask_console_text(text=plan_text.stdout, masks=masks).splitlines(), + "diagnostics": _diagnostics(result=plan_text, masks=masks), + } + build: CommandResult = run_sqb( + arguments=("build", *arguments), + project_dir=project_dir, + environment=environment, + python=options.python, + ) + record["build"] = _build_record(result=build, masks=masks) + state: dict[str, list[str]] = state_rows(project_dir=project_dir, masks=masks) + record["state"] = state_delta(before=previous_state, after=state) + previous_state = state + if plan_json.exit_code == 0 and not set(arguments) & _SELECTION_FLAGS: + models = plan_models(plan_json.stdout) + elif plan_json.exit_code != 0 and step.mutation is not None: + restore_project_files(project_dir=project_dir, files=saved) + record["reverted"] = True + source_tables = _source_tables(plan_output) or source_tables + if not options.keep_work: + shutil.rmtree(case_dir, ignore_errors=True) + return {"project": project.name, "steps": records} + + +def _prepared_project(*, project: PlannerProject, case_dir: Path) -> Path: + project_dir: Path = case_dir / PROJECT_DIRECTORY + shutil.rmtree(case_dir, ignore_errors=True) + if project.source_dir is not None: + _ = shutil.copytree( + project.source_dir, project_dir, ignore=shutil.ignore_patterns(*COPY_IGNORED) + ) + elif project.writer is not None: + project_dir.mkdir(parents=True) + project.writer(project_dir) + return project_dir + + +def _plan_json_record(*, result: CommandResult, masks: TextMasks) -> dict[str, object]: + try: + document: object = canonical_value(value=json.loads(result.stdout), masks=masks) + except json.JSONDecodeError: + document = mask_console_text(text=result.stdout, masks=masks).splitlines() + return { + "exit_code": result.exit_code, + "document": document, + "diagnostics": _diagnostics(result=result, masks=masks), + } + + +def _plan_capture(*, capture_dir: Path, masks: TextMasks) -> list[object]: + return [ + canonical_plan_capture(text=path.read_text(encoding="utf-8"), masks=masks) + for path in sorted(capture_dir.glob("*.json")) + ] + + +def _build_record(*, result: CommandResult, masks: TextMasks) -> dict[str, object]: + output: str = mask_console_text(text=result.stdout + result.stderr, masks=masks) + lines: list[str] = [line.strip() for line in output.splitlines() if line.strip()] + summary: list[str] = [line for line in lines if line.startswith(BUILD_SUMMARY_PREFIXES)] + return { + "exit_code": result.exit_code, + "summary": summary, + "diagnostics": sorted({line for line in lines if ERROR_LINE_PATTERN.match(line)}), + "last_line": lines[-1] if lines and result.exit_code != 0 and not summary else None, + } + + +def _diagnostics(*, result: CommandResult, masks: TextMasks) -> list[str]: + return [ + line.strip() + for line in mask_console_text(text=result.stderr, masks=masks).splitlines() + if ERROR_LINE_PATTERN.match(line) + ] + + +def _source_tables(plan_output: list[object]) -> tuple[str, ...]: + """Return the warehouse tables of the last captured plan's non-expression sources.""" + + last: dict[str, object] | None = as_json_object(plan_output[-1] if plan_output else None) + source_map: dict[str, object] | None = as_json_object( + None if last is None else last.get(_SOURCE_MAP_KEY) + ) + tables: list[str] = [] + for name, value in sorted((source_map or {}).items()): + entry: dict[str, object] | None = as_json_object(value) + if entry is None or entry.get("expression") is not None: + continue + table: str = str(entry.get("table") or name) + schema: object = entry.get("schema") + tables.append(table if schema is None else f"{schema}.{table}") + return tuple(tables) diff --git a/scripts/planner_differential/_helpers/running/report.py b/scripts/planner_differential/_helpers/running/report.py new file mode 100644 index 0000000000..9e1bd0af92 --- /dev/null +++ b/scripts/planner_differential/_helpers/running/report.py @@ -0,0 +1,38 @@ +"""Render planner differential results for terminals and CI logs.""" + +from __future__ import annotations + +from scripts.planner_differential.constants import MODE_RECORD +from scripts.planner_differential.models import ProjectDifference, ProjectResult + + +def format_result(result: ProjectResult) -> str: + """Render one project's verdict, harness error and every readable diff.""" + + status: str = "ERROR" if result.error else ("DIFF " if result.differences else "OK ") + lines: list[str] = [f"{status} {result.project} ({result.seconds:.1f}s)"] + if result.error: + lines.extend(f" {line}" for line in result.error.rstrip().splitlines()) + for difference in result.differences: + lines.append(_format_difference(difference)) + return "\n".join(lines) + + +def format_summary(*, results: list[ProjectResult], mode: str, seconds: float) -> str: + """Summarize the run; a difference or harness error reports FAILED.""" + + failed: list[ProjectResult] = [ + result for result in results if result.differences or result.error + ] + if not failed: + verb: str = "recorded" if mode == MODE_RECORD else "matched the baseline" + return f"Planner differential passed: {len(results)} projects {verb} in {seconds:.1f}s" + return ( + f"Planner differential FAILED: {len(failed)} of {len(results)} projects differ or " + f"errored in {seconds:.1f}s" + ) + + +def _format_difference(difference: ProjectDifference) -> str: + body: str = "\n".join(f" {line}" for line in difference.diff.splitlines()) + return f" - step {difference.step}: {difference.artifact}\n{body}" diff --git a/scripts/planner_differential/_helpers/running/run.py b/scripts/planner_differential/_helpers/running/run.py new file mode 100644 index 0000000000..7c891b83d4 --- /dev/null +++ b/scripts/planner_differential/_helpers/running/run.py @@ -0,0 +1,74 @@ +"""Run the corpus in parallel and compare or record each project.""" + +from __future__ import annotations + +import sys +import time +import traceback +from concurrent.futures import ThreadPoolExecutor + +from scripts.planner_differential._helpers.baseline.baseline import ( + compare_documents, + read_document, + write_document, +) +from scripts.planner_differential._helpers.running.project_run import project_slug, run_project +from scripts.planner_differential._helpers.running.report import format_result +from scripts.planner_differential.constants import ACTUAL_SUFFIX, MODE_RECORD +from scripts.planner_differential.models import PlannerProject, ProjectResult, RunOptions + + +def run_corpus(*, projects: list[PlannerProject], options: RunOptions) -> list[ProjectResult]: + """Run every project, printing each verdict as it completes.""" + + print( + f"Planner differential {options.mode}: {len(projects)} projects x " + f"{len(options.steps)} steps ({options.jobs} jobs), baseline {options.baseline_dir}", + file=sys.stderr, + ) + results: list[ProjectResult] = [] + with ThreadPoolExecutor(max_workers=options.jobs) as pool: + for result in pool.map( + lambda project: _project_result(project=project, options=options), projects + ): + results.append(result) + print(format_result(result)) + sys.stdout.flush() + return results + + +def _project_result(*, project: PlannerProject, options: RunOptions) -> ProjectResult: + started: float = time.monotonic() + slug: str = project_slug(project.name) + try: + document: dict[str, object] = run_project(project=project, options=options) + except Exception: # noqa: BLE001 - one project's harness failure must not hide the others + return ProjectResult( + project=project.name, + seconds=time.monotonic() - started, + differences=(), + error=traceback.format_exc(), + ) + if options.mode == MODE_RECORD: + _ = write_document(directory=options.baseline_dir, slug=slug, document=document) + return ProjectResult( + project=project.name, seconds=time.monotonic() - started, differences=() + ) + _ = write_document( + directory=options.baseline_dir.with_name(options.baseline_dir.name + ACTUAL_SUFFIX), + slug=slug, + document=document, + ) + expected: dict[str, object] | None = read_document(directory=options.baseline_dir, slug=slug) + if expected is None: + return ProjectResult( + project=project.name, + seconds=time.monotonic() - started, + differences=(), + error=f"no baseline for {project.name}; run with `record` first", + ) + return ProjectResult( + project=project.name, + seconds=time.monotonic() - started, + differences=compare_documents(expected=expected, actual=document), + ) diff --git a/scripts/planner_differential/_helpers/running/snapshot.py b/scripts/planner_differential/_helpers/running/snapshot.py new file mode 100644 index 0000000000..f7fa54e95a --- /dev/null +++ b/scripts/planner_differential/_helpers/running/snapshot.py @@ -0,0 +1,37 @@ +"""Save and restore a project's authored files around a mutation that breaks planning.""" + +from __future__ import annotations + +import fnmatch +from pathlib import Path + +from scripts.planner_differential.constants import COPY_IGNORED + + +def project_files(project_dir: Path) -> dict[str, bytes]: + """Return every authored file's bytes, keyed by its project-relative path.""" + + return { + path.relative_to(project_dir).as_posix(): path.read_bytes() + for path in sorted(project_dir.rglob("*")) + if path.is_file() and not _ignored(path.relative_to(project_dir)) + } + + +def restore_project_files(*, project_dir: Path, files: dict[str, bytes]) -> None: + """Rewrite the saved files and remove authored files added since they were saved.""" + + for relative in project_files(project_dir): + if relative not in files: + (project_dir / relative).unlink() + for relative, contents in files.items(): + path: Path = project_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_bytes(contents) + + +def _ignored(relative: Path) -> bool: + for part in relative.parts: + if any(fnmatch.fnmatch(part, pattern) for pattern in COPY_IGNORED): + return True + return False diff --git a/scripts/planner_differential/_helpers/steps/__init__.py b/scripts/planner_differential/_helpers/steps/__init__.py new file mode 100644 index 0000000000..cbc29d81a9 --- /dev/null +++ b/scripts/planner_differential/_helpers/steps/__init__.py @@ -0,0 +1 @@ +"""The mutation script applied to every project.""" diff --git a/scripts/planner_differential/_helpers/steps/arguments.py b/scripts/planner_differential/_helpers/steps/arguments.py new file mode 100644 index 0000000000..7f26204b59 --- /dev/null +++ b/scripts/planner_differential/_helpers/steps/arguments.py @@ -0,0 +1,57 @@ +"""Resolve a step's flag placeholders against its target model.""" + +from __future__ import annotations + +import duckdb + +from scripts.planner_differential._helpers.capturing.state_tables import database_files +from scripts.planner_differential._helpers.steps.relations import schema_and_name +from scripts.planner_differential.models import ModelFacts, PlannerStep, StepContext + +_MODEL_PLACEHOLDER: str = "{model}" +_START_PLACEHOLDER: str = "{start_cursor}" +_END_PLACEHOLDER: str = "{end_cursor}" + + +def step_arguments( + *, step: PlannerStep, context: StepContext, target: ModelFacts | None +) -> tuple[str, ...] | None: + """Return the step's plan and build flags, or None when a placeholder has no value.""" + + values: dict[str, str] = {} + if target is not None: + values[_MODEL_PLACEHOLDER] = target.name + if any(_START_PLACEHOLDER in argument for argument in step.arguments): + bounds: tuple[str, str] | None = ( + None if target is None else _cursor_bounds(context=context, target=target) + ) + if bounds is not None: + values[_START_PLACEHOLDER], values[_END_PLACEHOLDER] = bounds + resolved: list[str] = [] + for argument in step.arguments: + if argument.startswith("{") and argument not in values: + return None + resolved.append(values.get(argument, argument)) + return tuple(resolved) + + +def _cursor_bounds(*, context: StepContext, target: ModelFacts) -> tuple[str, str] | None: + """The minimum and maximum cursor values the target's relation holds.""" + + if target.cursor_column is None: + return None + schema, name = schema_and_name(target.qualified_name) + relation: str = f'"{schema}"."{name}"' + column: str = f'"{target.cursor_column}"' + for database in database_files(context.project_dir): + with duckdb.connect(str(database), read_only=True) as connection: + try: + found: tuple[object, ...] | None = connection.execute( + f"SELECT CAST(min({column}) AS VARCHAR), CAST(max({column}) AS VARCHAR) " + f"FROM {relation}" + ).fetchone() + except duckdb.CatalogException: + continue + if found is not None and found[0] is not None and found[1] is not None: + return (str(found[0]), str(found[1])) + return None diff --git a/scripts/planner_differential/_helpers/steps/model_file.py b/scripts/planner_differential/_helpers/steps/model_file.py new file mode 100644 index 0000000000..f233e20021 --- /dev/null +++ b/scripts/planner_differential/_helpers/steps/model_file.py @@ -0,0 +1,50 @@ +"""Edit a SQL model file's MODEL header and query as text.""" + +from __future__ import annotations + +import re + +from scripts.planner_differential.models import ModelFile + +_HEADER_START: re.Pattern[str] = re.compile(r"^\s*MODEL\s*\(", re.MULTILINE) +_HEADER_END: re.Pattern[str] = re.compile(r"^[ \t]*\)[ \t]*;[ \t]*$", re.MULTILINE) +_STATEMENT_SEPARATOR: str = ";" +_INLINE_HEADER: re.Pattern[str] = re.compile(r"\A\s*MODEL\s*\((?P[^;]*)\)\s*;") + + +def split_model_file(text: str) -> ModelFile | None: + """Split a model file into its MODEL header and one-statement query, or None.""" + + header: str = "" + start: re.Match[str] | None = _HEADER_START.match(text) + if start is not None: + end: re.Match[str] | None = _HEADER_END.search(text, start.end()) + inline: re.Match[str] | None = _INLINE_HEADER.match(text) + if end is not None: + header = text[: end.end()] + elif inline is not None: + header = inline.group(0) + else: + return None + query: str = text[len(header) :].strip().rstrip(";").strip() + if not query or _STATEMENT_SEPARATOR in query or _HEADER_START.match(query): + return None + return ModelFile(header=header, query=query) + + +def join_model_file(model_file: ModelFile) -> str: + """Return the file text for a header and query.""" + + prefix: str = f"{model_file.header}\n\n" if model_file.header else "" + return f"{prefix}{model_file.query}\n" + + +def add_header_property(*, header: str, line: str) -> str: + """Insert one `name value,` property at the start of the MODEL header.""" + + if not header: + return f"MODEL (\n {line}\n);" + start: re.Match[str] | None = _HEADER_START.match(header) + if start is None: + return header + return f"{header[: start.end()]}\n {line}{header[start.end() :]}" diff --git a/scripts/planner_differential/_helpers/steps/mutations.py b/scripts/planner_differential/_helpers/steps/mutations.py new file mode 100644 index 0000000000..eeee8094c7 --- /dev/null +++ b/scripts/planner_differential/_helpers/steps/mutations.py @@ -0,0 +1,291 @@ +"""The project mutations a step applies before planning.""" + +from __future__ import annotations + +import re +from collections.abc import Callable +from pathlib import Path + +import duckdb + +from scripts.planner_differential._helpers.capturing.state_tables import database_files +from scripts.planner_differential._helpers.running.commands import run_sqb +from scripts.planner_differential._helpers.steps.model_file import ( + add_header_property, + join_model_file, + split_model_file, +) +from scripts.planner_differential._helpers.steps.relations import schema_and_name +from scripts.planner_differential.constants import ( + ADDED_COLUMN_NAME, + ADDED_MODEL_NAME, + ADDED_MODEL_PATH, + EDITED_DESCRIPTION, + MUTATION_ADD_MODEL, + MUTATION_CONFIG, + MUTATION_DECLARE_COLUMN, + MUTATION_DROP_RELATION, + MUTATION_EDIT_QUERY, + MUTATION_MATERIALIZATION, + MUTATION_RENAME_COLUMN, + MUTATION_RENAME_MODEL, + MUTATION_SOURCE_GROWTH, + MUTATION_TAGS, + RENAMED_SUFFIX, + SOURCE_GROWTH_INTERVAL, +) +from scripts.planner_differential.models import ( + CommandResult, + ModelFacts, + ModelFile, + MutationResult, + StepContext, +) + +_TAGS_PATTERN: re.Pattern[str] = re.compile(r"^(\s*tags\s*\[)", re.MULTILINE) +_SCHEMA_CHANGE_KEY: str = "on_schema_change" +_SCHEMA_CHANGE_PATTERN: re.Pattern[str] = re.compile( + rf"^(\s*{_SCHEMA_CHANGE_KEY}\s+)(\w+)", re.MULTILINE +) +_APPEND_NEW_COLUMNS: str = "append_new_columns" +_SCHEMA_CHANGE_TOGGLE: dict[str, str] = { + _APPEND_NEW_COLUMNS: "sync_all_columns", + "sync_all_columns": _APPEND_NEW_COLUMNS, +} +_MATERIALIZED_PATTERN: re.Pattern[str] = re.compile(r"^(\s*materialized\s+)\w+", re.MULTILINE) +_COLUMNS_PATTERN: re.Pattern[str] = re.compile(r"^(\s*columns\s*\()", re.MULTILINE) +_EDIT_ALIAS: str = "planner_differential_edit" +_TAG: str = "planner_differential" +_TIME_TYPES: tuple[str, ...] = ("TIMESTAMP", "DATE") +_COLUMNS_SQL: str = ( + "SELECT column_name, data_type FROM information_schema.columns " + "WHERE table_schema = ? AND table_name = ? ORDER BY ordinal_position" +) +_SOURCE_COLUMNS_SQL: str = ( + "SELECT c.table_schema, c.column_name, c.data_type FROM information_schema.columns c " + "JOIN information_schema.tables t " + "ON t.table_schema = c.table_schema AND t.table_name = c.table_name " + "WHERE t.table_type = 'BASE TABLE' AND c.table_name = ? " + "AND (CAST(? AS VARCHAR) IS NULL OR c.table_schema = ?) " + "ORDER BY c.table_schema, c.ordinal_position" +) +_RELATION_TYPE_SQL: str = ( + "SELECT table_type FROM information_schema.tables WHERE table_schema = ? AND table_name = ?" +) +_VIEW_TYPE: str = "VIEW" + + +def apply_mutation(*, name: str, context: StepContext, target: ModelFacts) -> MutationResult: + """Apply the named mutation to the target model or its project.""" + + mutations: dict[str, Callable[..., MutationResult]] = { + MUTATION_EDIT_QUERY: _edit_query, + MUTATION_TAGS: _tags, + MUTATION_CONFIG: _config, + MUTATION_MATERIALIZATION: _materialization, + MUTATION_DECLARE_COLUMN: _declare_column, + MUTATION_RENAME_COLUMN: _rename_column, + MUTATION_ADD_MODEL: _add_model, + MUTATION_RENAME_MODEL: _rename_model, + MUTATION_DROP_RELATION: _drop_relation, + MUTATION_SOURCE_GROWTH: _source_growth, + } + return mutations[name](context=context, target=target) + + +def _edit_query(*, context: StepContext, target: ModelFacts) -> MutationResult: + return _rewrite_model( + context=context, + target=target, + rewrite=lambda model_file: ModelFile( + header=model_file.header, + query=f"SELECT *\nFROM (\n{model_file.query}\n) AS {_EDIT_ALIAS}", + ), + ) + + +def _tags(*, context: StepContext, target: ModelFacts) -> MutationResult: + def rewrite(model_file: ModelFile) -> ModelFile: + header: str = ( + _TAGS_PATTERN.sub(rf"\g<1>{_TAG}, ", model_file.header, count=1) + if _TAGS_PATTERN.search(model_file.header) + else add_header_property(header=model_file.header, line=f"tags [{_TAG}],") + ) + return ModelFile(header=header, query=model_file.query) + + return _rewrite_model(context=context, target=target, rewrite=rewrite) + + +def _config(*, context: StepContext, target: ModelFacts) -> MutationResult: + def rewrite(model_file: ModelFile) -> ModelFile: + found: re.Match[str] | None = _SCHEMA_CHANGE_PATTERN.search(model_file.header) + value: str = ( + _SCHEMA_CHANGE_TOGGLE.get(found.group(2), _APPEND_NEW_COLUMNS) + if found is not None + else _APPEND_NEW_COLUMNS + ) + header: str = ( + _SCHEMA_CHANGE_PATTERN.sub(rf"\g<1>{value}", model_file.header, count=1) + if found is not None + else add_header_property( + header=model_file.header, line=f"{_SCHEMA_CHANGE_KEY} {value}," + ) + ) + return ModelFile(header=header, query=model_file.query) + + return _rewrite_model(context=context, target=target, rewrite=rewrite) + + +def _materialization(*, context: StepContext, target: ModelFacts) -> MutationResult: + def rewrite(model_file: ModelFile) -> ModelFile: + header: str = ( + _MATERIALIZED_PATTERN.sub(r"\g<1>view", model_file.header, count=1) + if _MATERIALIZED_PATTERN.search(model_file.header) + else add_header_property(header=model_file.header, line="materialized view,") + ) + return ModelFile(header=header, query=model_file.query) + + return _rewrite_model(context=context, target=target, rewrite=rewrite) + + +def _declare_column(*, context: StepContext, target: ModelFacts) -> MutationResult: + declaration: str = f'{ADDED_COLUMN_NAME} (description "{EDITED_DESCRIPTION}"),' + columns: list[str] = [ + f'{_EDIT_ALIAS}."{column}"' + for column, _ in _relation_columns(context=context, target=target) + ] + if not columns: + return MutationResult(applied=False, note=f"no built relation for {target.name}") + + def rewrite(model_file: ModelFile) -> ModelFile: + header: str = ( + _COLUMNS_PATTERN.sub(rf"\g<1>\n {declaration}", model_file.header, count=1) + if _COLUMNS_PATTERN.search(model_file.header) + else add_header_property(header=model_file.header, line=f"columns ({declaration}),") + ) + return ModelFile( + header=header, + query=( + f"SELECT {', '.join(columns)}, CAST(1 AS INTEGER) AS {ADDED_COLUMN_NAME}\n" + f"FROM (\n{model_file.query}\n) AS {_EDIT_ALIAS}" + ), + ) + + return _rewrite_model(context=context, target=target, rewrite=rewrite) + + +def _rename_column(*, context: StepContext, target: ModelFacts) -> MutationResult: + path: Path = context.project_dir / target.relative_path + model_file: ModelFile | None = split_model_file(path.read_text(encoding="utf-8")) + header: str = "" if model_file is None else model_file.header + columns: list[str] = [ + column + for column, _ in _relation_columns(context=context, target=target) + if column != target.cursor_column + and column != ADDED_COLUMN_NAME + and re.search(rf"\b{re.escape(column)}\b", header) is None + ] + if not columns: + return MutationResult(applied=False, note=f"no renameable column in {target.name}") + column: str = columns[-1] + return _sqb_mutation( + context=context, + arguments=("rename", f"{target.name}.{column}", f"{column}{RENAMED_SUFFIX}"), + ) + + +def _add_model(*, context: StepContext, target: ModelFacts) -> MutationResult: + path: Path = context.project_dir / ADDED_MODEL_PATH + _ = path.write_text( + f'MODEL (\n materialized table,\n description "{EDITED_DESCRIPTION}",\n);\n\n' + f'SELECT *\nFROM __ref("{target.name}")\n', + encoding="utf-8", + newline="\n", + ) + return MutationResult(applied=True, note=f"added {ADDED_MODEL_NAME} reading {target.name}") + + +def _rename_model(*, context: StepContext, target: ModelFacts) -> MutationResult: + return _sqb_mutation( + context=context, arguments=("rename", target.name, f"{target.name}{RENAMED_SUFFIX}") + ) + + +def _drop_relation(*, context: StepContext, target: ModelFacts) -> MutationResult: + schema, name = schema_and_name(target.qualified_name) + for database in database_files(context.project_dir): + with duckdb.connect(str(database)) as connection: + found: tuple[object, ...] | None = connection.execute( + _RELATION_TYPE_SQL, [schema, name] + ).fetchone() + if found is None: + continue + kind: str = "VIEW" if found[0] == _VIEW_TYPE else "TABLE" + _ = connection.execute(f'DROP {kind} "{schema}"."{name}"') + return MutationResult(applied=True, note=f"dropped {kind.lower()} {schema}.{name}") + return MutationResult(applied=False, note=f"no relation {target.qualified_name}") + + +def _source_growth(*, context: StepContext, target: ModelFacts) -> MutationResult: + _ = target + grown: list[str] = [] + for source in context.source_tables: + schema: str | None = source.rpartition(".")[0] or None + name: str = source.rpartition(".")[2] + for database in database_files(context.project_dir): + with duckdb.connect(str(database)) as connection: + columns: list[tuple[str, str, str]] = connection.execute( + _SOURCE_COLUMNS_SQL, [name, schema, schema] + ).fetchall() + time_column: tuple[str, str, str] | None = next( + (column for column in columns if column[2].startswith(_TIME_TYPES)), None + ) + if time_column is None: + continue + table_schema, column, column_type = time_column + relation: str = f'"{table_schema}"."{name}"' + _ = connection.execute( + f'INSERT INTO {relation} SELECT * REPLACE (CAST("{column}" + ' + f'{SOURCE_GROWTH_INTERVAL} AS {column_type}) AS "{column}") ' + f'FROM {relation} WHERE "{column}" = (SELECT max("{column}") FROM {relation})' + ) + grown.append(f"{table_schema}.{name}.{column}") + if not grown: + return MutationResult(applied=False, note="no source table with a time column") + return MutationResult(applied=True, note=f"grew {', '.join(grown)}") + + +def _rewrite_model( + *, context: StepContext, target: ModelFacts, rewrite: Callable[[ModelFile], ModelFile] +) -> MutationResult: + path: Path = context.project_dir / target.relative_path + model_file: ModelFile | None = split_model_file(path.read_text(encoding="utf-8")) + if model_file is None: + return MutationResult(applied=False, note=f"cannot edit {target.relative_path}") + _ = path.write_text(join_model_file(rewrite(model_file)), encoding="utf-8", newline="\n") + return MutationResult(applied=True, note=f"edited {target.relative_path}") + + +def _sqb_mutation(*, context: StepContext, arguments: tuple[str, ...]) -> MutationResult: + result: CommandResult = run_sqb( + arguments=arguments, + project_dir=context.project_dir, + environment=context.environment, + python=context.python, + ) + return MutationResult( + applied=result.exit_code == 0, + note=f"sqb {' '.join(arguments)} exited {result.exit_code}", + ) + + +def _relation_columns(*, context: StepContext, target: ModelFacts) -> list[tuple[str, str]]: + schema, name = schema_and_name(target.qualified_name) + for database in database_files(context.project_dir): + with duckdb.connect(str(database), read_only=True) as connection: + columns: list[tuple[str, str]] = connection.execute( + _COLUMNS_SQL, [schema, name] + ).fetchall() + if columns: + return columns + return [] diff --git a/scripts/planner_differential/_helpers/steps/relations.py b/scripts/planner_differential/_helpers/steps/relations.py new file mode 100644 index 0000000000..aa72d48431 --- /dev/null +++ b/scripts/planner_differential/_helpers/steps/relations.py @@ -0,0 +1,12 @@ +"""Split relation names the plan reports.""" + +from __future__ import annotations + +_DEFAULT_SCHEMA: str = "main" + + +def schema_and_name(qualified_name: str) -> tuple[str, str]: + """Return a `schema.name` or `database.schema.name` relation's schema and name.""" + + parts: list[str] = qualified_name.split(".") + return (parts[-2] if len(parts) > 1 else _DEFAULT_SCHEMA, parts[-1]) diff --git a/scripts/planner_differential/_helpers/steps/targets.py b/scripts/planner_differential/_helpers/steps/targets.py new file mode 100644 index 0000000000..802783ee4c --- /dev/null +++ b/scripts/planner_differential/_helpers/steps/targets.py @@ -0,0 +1,107 @@ +"""Choose the model each step mutates or selects from the previous plan.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +from scripts.planner_differential._helpers.steps.model_file import split_model_file +from scripts.planner_differential.constants import ( + ADDED_MODEL_NAME, + ROLE_ANY, + ROLE_CURSOR, + ROLE_INCREMENTAL, + ROLE_LEAF, + ROLE_MICROBATCH, + ROLE_TABLE, +) +from scripts.planner_differential.models import ModelFacts + +_MATERIALIZATION_TABLE: str = "table" +_MATERIALIZATION_INCREMENTAL: str = "incremental" +_MICROBATCH_MODE: str = "microbatch" +_SQL_SUFFIX: str = ".sql" +_NAME_KEY: str = "name" +_TARGET_DIRECTORY: str = "target" +_SEARCHED_SUFFIXES: frozenset[str] = frozenset({".sql", ".yml", ".yaml", ".py", ".toml"}) + + +def plan_models(plan_json: str) -> tuple[ModelFacts, ...]: + """Return the models a `sqb plan --json` document lists, or none when it is not JSON.""" + + try: + document: object = json.loads(plan_json) + except json.JSONDecodeError: + return () + entries: object = document.get("models") if isinstance(document, dict) else None + if not isinstance(entries, list): + return () + return tuple( + ModelFacts( + name=str(entry[_NAME_KEY]), + relative_path=str(entry.get("relative_path") or ""), + materialization=str(entry.get("materialization_type") or ""), + qualified_name=str(entry.get("qualified_name") or ""), + cursor_column=entry.get("cursor_column"), + microbatch=entry.get("incremental_mode") == _MICROBATCH_MODE, + ) + for entry in entries + if isinstance(entry, dict) and _NAME_KEY in entry + ) + + +def role_target( + *, role: str, index: int, models: tuple[ModelFacts, ...], project_dir: Path +) -> ModelFacts | None: + """Return the role's `index`-th candidate (wrapping), or None when it has none.""" + + candidates: list[ModelFacts] = sorted( + ( + model + for model in models + if model.name != ADDED_MODEL_NAME + and _editable(model=model, project_dir=project_dir) + and _has_role(model=model, role=role, project_dir=project_dir) + ), + key=lambda model: model.name, + ) + return candidates[index % len(candidates)] if candidates else None + + +def _editable(*, model: ModelFacts, project_dir: Path) -> bool: + path: Path = project_dir / model.relative_path + return ( + model.relative_path.endswith(_SQL_SUFFIX) + and path.is_file() + and split_model_file(path.read_text(encoding="utf-8")) is not None + ) + + +def _has_role(*, model: ModelFacts, role: str, project_dir: Path) -> bool: + if role == ROLE_TABLE: + return model.materialization == _MATERIALIZATION_TABLE + if role == ROLE_INCREMENTAL: + return model.materialization == _MATERIALIZATION_INCREMENTAL + if role == ROLE_MICROBATCH: + return model.microbatch + if role == ROLE_CURSOR: + return model.cursor_column is not None + if role == ROLE_LEAF: + return model.materialization == _MATERIALIZATION_TABLE and not _referenced( + model=model, project_dir=project_dir + ) + return role == ROLE_ANY + + +def _referenced(*, model: ModelFacts, project_dir: Path) -> bool: + pattern: re.Pattern[str] = re.compile(rf"\b{re.escape(model.name)}\b") + own: Path = project_dir / model.relative_path + return any( + path != own + and path.suffix in _SEARCHED_SUFFIXES + and _TARGET_DIRECTORY not in path.relative_to(project_dir).parts + and pattern.search(path.read_text(encoding="utf-8", errors="replace")) is not None + for path in sorted(project_dir.rglob("*")) + if path.is_file() + ) diff --git a/scripts/planner_differential/constants.py b/scripts/planner_differential/constants.py new file mode 100644 index 0000000000..2ee124ef40 --- /dev/null +++ b/scripts/planner_differential/constants.py @@ -0,0 +1,144 @@ +"""Constants for the planner differential.""" + +from __future__ import annotations + +import re + +from scripts.planner_differential.models import PlannerStep + +PLAN_CAPTURE_ENV_VAR: str = "SQLBUILD_PLANNER_CAPTURE_DIR" +MODE_RECORD: str = "record" +MODE_CHECK: str = "check" +MODES: tuple[str, ...] = (MODE_RECORD, MODE_CHECK) +DEFAULT_BASELINE_DIR: str = "/tmp/sqlbuild-planner-differential/baseline" +DEFAULT_SEED_COUNT: int = 6 +DEFAULT_JOBS: int = 2 +BASELINE_SUFFIX: str = ".json" +ACTUAL_SUFFIX: str = ".actual" +FIXTURE_ROOT: str = "tests/e2e/fixtures" +EXAMPLE_ROOT: str = "website/examples" +PROJECT_CONFIG_FILE: str = "sqlbuild_project.toml" +PROJECT_DIRECTORY: str = "project" +CAPTURE_DIRECTORY: str = "capture" +DUCKDB_ADAPTER: str = "duckdb" +DUCKDB_SUFFIX: str = ".duckdb" +COMMAND_TIMEOUT_SECONDS: float = 600.0 +SQB_ENTRY: str = "import sys; from sqlbuild.cli.entry.main.entry import main; sys.exit(main())" +EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset( + {"VIRTUAL_ENV", PLAN_CAPTURE_ENV_VAR, "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR"} +) +EXCLUDED_ENVIRONMENT_PREFIX: str = "DBT_" +COPY_IGNORED: tuple[str, ...] = ( + "target", + "logs", + "__pycache__", + ".sqlbuild", + "*.duckdb", + "*.duckdb.wal", +) + +STATE_TABLE_PREFIX: str = "_sqlbuild" +ADDED_MODEL_NAME: str = "planner_differential_added" +ADDED_MODEL_PATH: str = "models/planner_differential_added.sql" +ADDED_COLUMN_NAME: str = "planner_differential_flag" +RENAMED_SUFFIX: str = "_renamed" +EDITED_DESCRIPTION: str = "Edited by the planner differential." +SOURCE_GROWTH_INTERVAL: str = "INTERVAL 1 DAY" +MAX_DIFF_LINES: int = 60 + +ROLE_TABLE: str = "table" +ROLE_INCREMENTAL: str = "incremental" +ROLE_MICROBATCH: str = "microbatch" +ROLE_CURSOR: str = "cursor" +ROLE_LEAF: str = "leaf" +ROLE_ANY: str = "any" + +MUTATION_EDIT_QUERY: str = "edit_query" +MUTATION_TAGS: str = "tags" +MUTATION_CONFIG: str = "config" +MUTATION_MATERIALIZATION: str = "materialization" +MUTATION_DECLARE_COLUMN: str = "declare_column" +MUTATION_RENAME_COLUMN: str = "rename_column" +MUTATION_ADD_MODEL: str = "add_model" +MUTATION_RENAME_MODEL: str = "rename_model" +MUTATION_DROP_RELATION: str = "drop_relation" +MUTATION_SOURCE_GROWTH: str = "source_growth" + +SELECT_MODEL_ARGUMENTS: tuple[str, ...] = ("--select", "{model}") +STEPS: tuple[PlannerStep, ...] = ( + PlannerStep(name="initial"), + PlannerStep(name="no-op"), + PlannerStep(name="edit-query", mutation=MUTATION_EDIT_QUERY, role=ROLE_TABLE), + PlannerStep(name="tag-change", mutation=MUTATION_TAGS, role=ROLE_TABLE, index=1), + PlannerStep(name="config-change", mutation=MUTATION_CONFIG, role=ROLE_INCREMENTAL, index=1), + PlannerStep(name="declared-column", mutation=MUTATION_DECLARE_COLUMN, role=ROLE_INCREMENTAL), + PlannerStep(name="column-rename", mutation=MUTATION_RENAME_COLUMN, role=ROLE_INCREMENTAL), + PlannerStep(name="add-model", mutation=MUTATION_ADD_MODEL, role=ROLE_TABLE), + PlannerStep(name="rename-model", mutation=MUTATION_RENAME_MODEL, role=ROLE_LEAF), + PlannerStep(name="drop-relation", mutation=MUTATION_DROP_RELATION, role=ROLE_TABLE), + PlannerStep(name="source-growth", mutation=MUTATION_SOURCE_GROWTH, role=ROLE_CURSOR), + PlannerStep( + name="max-microbatches", role=ROLE_MICROBATCH, arguments=("--max-microbatches", "1") + ), + PlannerStep( + name="cursor-override", + role=ROLE_CURSOR, + arguments=( + *SELECT_MODEL_ARGUMENTS, + "--start-cursor", + "{start_cursor}", + "--end-cursor", + "{end_cursor}", + ), + ), + PlannerStep( + name="exclude-edited", + mutation=MUTATION_EDIT_QUERY, + role=ROLE_TABLE, + index=1, + arguments=("--exclude", "{model}"), + ), + PlannerStep(name="select-edited", role=ROLE_TABLE, index=1, arguments=SELECT_MODEL_ARGUMENTS), + PlannerStep( + name="materialization-change", mutation=MUTATION_MATERIALIZATION, role=ROLE_TABLE, index=2 + ), + PlannerStep(name="full-refresh", arguments=("--full-refresh",)), + PlannerStep(name="final-no-op"), +) + +PROJECT_PATH_MASK: str = "" +WORK_PATH_MASK: str = "" +RECENT_TIMESTAMP_MASK: str = "" +VOLATILE_VALUE_MASK: str = "" +ELAPSED_TIME_PATTERN: re.Pattern[str] = re.compile(r"\b\d+(?:\.\d+)?\s?m?s\b") +ELAPSED_TIME_MASK: str = "" +ISO_TIMESTAMP_PATTERN: re.Pattern[str] = re.compile( + r"\b(\d{4}-\d{2}-\d{2})[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:?\d{2}|Z)?" +) +COMPACT_TIMESTAMP_PATTERN: re.Pattern[str] = re.compile(r"(? int: + """Run the corpus; in check mode exit 1 when any project differs from its baseline.""" + + args: argparse.Namespace = parse_args(argv) + projects: list[PlannerProject] = filter_corpus( + projects=build_corpus( + repo_root=Path(__file__).resolve().parents[3], + seed_start=args.seed_start, + seeds=args.seeds, + ), + names=tuple(args.projects), + limit=args.limit, + ) + started: float = time.monotonic() + with tempfile.TemporaryDirectory(prefix="sqlbuild-planner-differential-") as temporary: + results: list[ProjectResult] = run_corpus( + projects=projects, + options=run_options(args=args, work_dir=args.work_dir or Path(temporary)), + ) + print(format_summary(results=results, mode=args.mode, seconds=time.monotonic() - started)) + return 1 if any(result.differences or result.error for result in results) else 0 diff --git a/scripts/planner_differential/models.py b/scripts/planner_differential/models.py new file mode 100644 index 0000000000..2a0d8704f4 --- /dev/null +++ b/scripts/planner_differential/models.py @@ -0,0 +1,116 @@ +"""Typed records for the planner differential.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from datetime import date +from pathlib import Path + + +@dataclass(frozen=True) +class PlannerProject: + """One project the mutation script runs on.""" + + name: str + source_dir: Path | None = None + writer: Callable[[Path], None] | None = None + + +@dataclass(frozen=True) +class PlannerStep: + """One mutation of a `role` model (candidate `index`), then plan and build with `arguments`.""" + + name: str + mutation: str | None = None + role: str | None = None + index: int = 0 + arguments: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ModelFacts: + """What the previous plan says about one model, for choosing step targets.""" + + name: str + relative_path: str + materialization: str + qualified_name: str + cursor_column: str | None + microbatch: bool + + +@dataclass(frozen=True) +class ModelFile: + """A SQL model file's MODEL header (possibly empty) and its single query.""" + + header: str + query: str + + +@dataclass(frozen=True) +class StepContext: + """Where a step runs and the models the previous plan reported.""" + + project_dir: Path + models: tuple[ModelFacts, ...] + source_tables: tuple[str, ...] + environment: dict[str, str] + python: Path + + +@dataclass(frozen=True) +class MutationResult: + """Whether a mutation applied, and a stable note describing what it changed.""" + + applied: bool + note: str + + +@dataclass(frozen=True) +class TextMasks: + """Run-specific text to mask: path prefixes by their mask, and the run's date.""" + + replacements: tuple[tuple[str, str], ...] + today: date + + +@dataclass(frozen=True) +class CommandResult: + """One sqb invocation's exit code and output.""" + + exit_code: int + stdout: str + stderr: str + + +@dataclass(frozen=True) +class RunOptions: + """Resolved harness settings.""" + + work_dir: Path + jobs: int + python: Path + mode: str + baseline_dir: Path + keep_work: bool + steps: tuple[PlannerStep, ...] + + +@dataclass(frozen=True) +class ProjectDifference: + """One artifact of one step that differs from the baseline, with a readable diff.""" + + step: str + artifact: str + diff: str + + +@dataclass(frozen=True) +class ProjectResult: + """A project's verdict against the baseline.""" + + project: str + seconds: float + differences: tuple[ProjectDifference, ...] + error: str | None = None diff --git a/scripts/run_planner_differential.py b/scripts/run_planner_differential.py new file mode 100644 index 0000000000..ad5996f687 --- /dev/null +++ b/scripts/run_planner_differential.py @@ -0,0 +1,15 @@ +"""Direct wrapper for the planner differential harness.""" + +from __future__ import annotations + +from scripts.planner_differential.main.differential import run_planner_differential + + +def main(argv: list[str] | None = None) -> int: + """Record or check stateful planner outputs of the corpus across a mutation script.""" + + return run_planner_differential(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index bb9aee70a3..b2116bd81e 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5,7 +5,7 @@ from contextlib import AbstractContextManager from contextvars import Token from datetime import date, datetime from pathlib import Path, PurePath -from typing import Any, Literal, TextIO, TypedDict, overload +from typing import Any, TextIO, TypedDict from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter @@ -62,7 +62,13 @@ from sqlbuild.compiler.auditing.models import ( MeasurementContract, MeasurementThresholds, ) -from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditOutcome, AuditSeverity +from sqlbuild.compiler.auditing.types import ( + AuditAttachmentKind, + AuditEvaluationMode, + AuditOutcome, + AuditRunScope, + AuditSeverity, +) from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.models import ( CompileAdapterContext, @@ -71,6 +77,7 @@ from sqlbuild.compiler.compile.models import ( CompiledAudit, CompiledFunction, CompiledLineageColumnFact, + CompiledLineageSourceFact, CompiledModel, CompiledObjectKey, CompiledProject, @@ -87,18 +94,22 @@ from sqlbuild.compiler.compile.models import ( CompileSqlFunctionInput, CompileSqlReference, CompileSqlScenarioInput, + CompileSqlTestCte, CompileSqlTestInput, CursorInputRoles, DeclarationResolutionContext, DeclarationScopeBuild, DeclarationScopeResolver, DynamicColumnContractProof, + ExpansionSpan, + FixtureColumnInference, FunctionArgument, FunctionReturnColumn, InferredColumn, LoadedMacro, MacroContext, MacroExpansionResult, + MappedOffset, ModelInputBuildContext, NativeModelConfigInputs, NativeModelConfigSession, @@ -168,6 +179,9 @@ from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullabili from sqlbuild.compiler.migrations.models import MigrationRelation from sqlbuild.compiler.migrations.types import OldNameViewEventType from sqlbuild.compiler.pipeline.models import ( + ClonePipelineConnection, + ClonePipelineOptions, + ClonePipelineResult, CompiledProjectPhaseResult, CompilePipelineOptions, CompilePipelineResult, @@ -176,20 +190,40 @@ from sqlbuild.compiler.pipeline.models import ( PythonRunPlanOutputs, StaticCommandContext, ) +from sqlbuild.compiler.planner.classes.migration_state_inspection import ( + MigrationStateInspection, +) from sqlbuild.compiler.planner.models import ( AuditPlanEntry, + ChangeDetectionResult, CursorBounds, DeferralInputs, FutureCursorSafetyEvidence, MaximumStartSafetyEvidence, ModelPlanEntry, + PlannerChangeResults, + PlannerResolvedActions, + PlannerRuntime, + PlannerScope, PlanOutput, + PlanWarning, + ReferenceRename, + SchemaAction, + SchemaFinding, SqlTestPlanEntry, + WarehouseSnapshot, +) +from sqlbuild.compiler.planner.types import ( + CursorGrain, + CursorType, + MaterializationType, + OnSchemaChange, + PlanAction, + PlanReason, ) -from sqlbuild.compiler.planner.types import CursorGrain +from sqlbuild.compiler.project_assembly.models import NativeProjectResources from sqlbuild.compiler.python_nodes.models import ( PythonNodeGraph, - PythonNodeIdentity, PythonSqlRunLifecyclePlan, PythonSqlRunSelection, ) @@ -240,7 +274,7 @@ from sqlbuild.compiler.sql_test_glue.models import ( NativeSqlTestPlanningRequest, ) from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestPlanRow -from sqlbuild.cost.models import CostRunRecord +from sqlbuild.cost.models import CostRunRecord, StatementExecutionTelemetry from sqlbuild.cursor_algebra.models import ( AlignedInterval, DateValue, @@ -249,7 +283,11 @@ from sqlbuild.cursor_algebra.models import ( ) from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult +from sqlbuild.executor.build.models import ( + BuildExecutionResult, + SchedulerState, + SeedExecutionResult, +) from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult from sqlbuild.executor.diff.models import DiffExecutionResult from sqlbuild.executor.janitor.models import JanitorExecutionResult, JanitorPlan @@ -279,9 +317,8 @@ from sqlbuild.integrations.dbt.models import ( ) from sqlbuild.integrations.dbt.types import DbtInteropCommand from sqlbuild.lint.models import HeaderSpan, LintBody, LintConfig, LintRunResult, LintViolation -from sqlbuild.lint.types import RelationKeyRow +from sqlbuild.lint.types import RelationKeyRow, RelationKeys from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter from sqlbuild.presentation.models import DisplayOptions from sqlbuild.presentation.types import CompletionState from sqlbuild.python_nodes.models import SqlResourceRef @@ -294,6 +331,7 @@ from sqlbuild.spec.contracts.models import ( DbtConfig, DefaultsConfig, ExecutionLimitsConfig, + LoaderDestinationParts, LocalConfig, LocalDbtConfig, ProjectConfig, @@ -311,12 +349,13 @@ from sqlbuild.spec.contracts.models import ( TargetConfig, ) from sqlbuild.spec.contracts.types import ( + MicrobatchLimitAction, SourceFreshnessStrategy, SourceFreshnessValueKind, WarehouseGroup, ) -from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import StateSqlValueType +from sqlbuild.sql_values.models import SqlValue, SqlValueLimits +from sqlbuild.sql_values.types import CollectionRendering, StateSqlValueType BUILD_IDENTITY: str STAGE_CAPTURE_OMITTED_ATTRIBUTES: dict[str, frozenset[str]] @@ -1106,26 +1145,38 @@ class NativeModelLoop: def relation_key_row( kind: str, name: str, values: dict[str, object], entry: object ) -> RelationKeyRow: ... -def declared_relation_keys(rows: list[RelationKeyRow]) -> dict[str, list[list[str]]]: ... +def declared_relation_keys( + *, + model_rows: Iterable[RelationKeyRow], + seeds: Iterable[SchemaSeedEntry], + sources: Iterable[SourceEntry], +) -> RelationKeys: ... +def compiled_relation_keys(project: CompiledProjectView) -> RelationKeys: ... +def build_lint_expansion_context( + *, + project_dir: Path, + value_renderer: TypedSqlValueRenderer | None = None, + discovered_inputs: DiscoveredProjectInputs | None = None, + declaration_scope: DeclarationScopeBuild | None = None, + static_declaration_scope: DeclarationScopeBuild | None = None, +) -> SqlExpansionContext: ... +def write_atomically(*, path: Path, contents: str) -> None: ... def build_dag_json( *, graph: ProjectGraph, project_name: str, python_graph: PythonNodeGraph | None = None ) -> str: ... -class _ManifestFacts(TypedDict): - project_name: str - adapter_type: str - generated_at: str - hook_identities: Mapping[str, PythonNodeIdentity] - class NativeManifest: def contents(self) -> bytes: ... def build_manifest( *, project: CompiledProjectView, - plan_output: PlanOutput | None, - graph: NativeProjectGraph, - facts: _ManifestFacts, + plan_output: PlanOutput | None = None, + project_name: str, + adapter_type: str, + graph: NativeProjectGraph | None = None, + upstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]] | None = None, + downstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]] | None = None, ) -> NativeManifest: ... class NativeProjectGraph: @@ -1744,6 +1795,13 @@ def write_execution_header( ) -> None: ... def format_expectation_name(model_name: str) -> str: ... def format_expectation_detail(step_result: StepResult) -> str: ... +def format_build_footer( + *, + result: BuildExecutionResult, + elapsed: float, + use_color: bool, + python_node_results: tuple[PythonNodeExecutionResult, ...] | None = None, +) -> str: ... def format_build_execution_json( *, result: BuildExecutionResult, @@ -1934,9 +1992,12 @@ def apply_lint_suppressions( allow_suppressions: bool, ) -> list[LintViolation]: ... def load_rules_config(*, project_dir: Path) -> RulesConfig: ... -def selected_codes_json(request_json: str) -> str: ... -def render_owned_skill(content: str, input_fingerprint: str) -> str: ... -def skill_freshness(content: str | None, input_fingerprint: str) -> str: ... +def selected_rule_codes( + *, config: RulesConfig, custom_payloads: list[dict[str, object]] +) -> list[str]: ... +def rule_selected(*, config: RulesConfig, code: str) -> bool: ... +def render_owned_skill(*, content: str, input_fingerprint: str) -> str: ... +def skill_freshness(*, content: str | None, input_fingerprint: str) -> str: ... def match_model_headers(contents: list[str]) -> list[tuple[int, int, int] | None]: ... def parse_model_headers( headers: list[str], @@ -2271,9 +2332,36 @@ def declared_relations( def externally_described_models( inputs: DiscoveredProjectInputs, model_schemas: dict[str, ModelSchemaDeclaration] ) -> list[Path]: ... -def resolve_required_columns( - query_sql: str, references: list[tuple[str, str]], dialect: str | None -) -> list[tuple[str, str, str]]: ... +def analyze_resolved_column_reads( + *, query_sql: str, references: Sequence[CompileSqlReference], dialect: str | None +) -> tuple[CompiledLineageSourceFact, ...]: ... +def build_effective_target_config( + *, discovered_inputs: DiscoveredProjectInputs, selected_target: str | None = None +) -> TargetConfig | None: ... +def build_effective_connection_config( + *, + discovered_inputs: DiscoveredProjectInputs, + selected_target: str | None = None, + cli_vars: dict[str, object] | None = None, +) -> dict[str, object]: ... +def build_effective_runtime_config( + *, + discovered_inputs: DiscoveredProjectInputs, + selected_target: str | None = None, + cli_vars: dict[str, object] | None = None, +) -> tuple[str | None, dict[str, object], str]: ... +def build_sql_expansion_context( + *, + project_dir: Path, + value_renderer: TypedSqlValueRenderer, + cli_vars: dict[str, object] | None = None, + discovered_inputs: DiscoveredProjectInputs | None = None, + declaration_scope: DeclarationScopeBuild | None = None, + static_declaration_scope: DeclarationScopeBuild | None = None, +) -> SqlExpansionContext: ... +def build_effective_settings_config( + *, discovered_inputs: DiscoveredProjectInputs +) -> SettingsConfig: ... def public_declaration_indexes( inputs: DiscoveredProjectInputs, ) -> tuple[dict[str, EnumDeclaration], dict[str, ConstantDeclaration]]: ... @@ -2392,64 +2480,76 @@ class NativeScopeIndex: def index_with_relationships(self) -> ScopeIndex: ... def open_native_scope_index( + *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: Mapping[str, LoadedMacro], - static_macros: tuple[StaticMacroExport, ...], - /, + static_macros: tuple[StaticMacroExport, ...] = (), ) -> NativeScopeIndex: ... -def scope_placement_validated_index( +def build_scope_index( + *, + discovered_inputs: DiscoveredProjectInputs, + loaded_macros: Mapping[str, LoadedMacro] | None = None, + static_macros: tuple[StaticMacroExport, ...] = (), +) -> ScopeIndex: ... +def get_placement_validated_scope_index( + *, index: ScopeIndex, - enforce_placement: bool, - compile_usages: tuple[UsageRecord, ...] | None, + enforce_placement: bool = True, + compile_usages: tuple[UsageRecord, ...] | None = None, ) -> ScopeIndex: ... -def scope_relocated_declarations( - index: ScopeIndex, resource: ResourceIdentity, destination: str +def relocate_declarations_for_move( + *, index: ScopeIndex, resource: ResourceIdentity, destination: str ) -> tuple[DeclarationRecord, ...] | None: ... +def validate_scope_index(*, index: ScopeIndex) -> None: ... class NativeScopeLookup: ... -def scope_query_target(lookup: ScopeLookup, target: ScopeQueryTarget, /) -> ScopeTargetQuery: ... -@overload -def scope_resolve_visibility( - lookup: ScopeLookup, target: ScopeQueryTarget, with_reasons: Literal[True], / +def query_scope_target(*, lookup: ScopeLookup, target: ScopeQueryTarget) -> ScopeTargetQuery: ... +def resolve_scope_visibility( + *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath ) -> VisibilityResolution: ... -@overload -def scope_resolve_visibility( - lookup: ScopeLookup, target: ScopeQueryTarget, with_reasons: Literal[False], / +def resolve_scope_declaration_visibility( + *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath ) -> DeclarationVisibility: ... -def scope_resolve_path_visibility( - lookup: ScopeLookup, path: str | PurePath, / +def resolve_scope_path_visibility( + *, lookup: ScopeLookup, path: str | PurePath ) -> tuple[tuple[DeclarationRecord, ...], tuple[DeclarationRecord, ...]]: ... -def build_scope_lookup(index: ScopeIndex) -> ScopeLookup: ... -def scope_report( +def build_scope_lookup(*, index: ScopeIndex) -> ScopeLookup: ... +def query_scope_report( + *, lookup: ScopeLookup, - request: tuple[ScopeQueryTarget | None, str | PurePath | None, bool, ScopeReportFilters | None], - /, + target: str | PurePath | ResourceIdentity | DeclarationIdentity | None = None, + at: str | PurePath | None = None, + directory: bool = False, + filters: ScopeReportFilters | None = None, ) -> ScopeReport: ... -def scope_explain( +def explain_scope_declaration( + *, lookup: ScopeLookup, - request: tuple[ - str | DeclarationIdentity, - str | PurePath | ResourceIdentity | None, - str | PurePath | None, - bool, - ], - /, + declaration: str | DeclarationIdentity, + target: str | PurePath | ResourceIdentity | None = None, + at: str | PurePath | None = None, + directory: bool = False, ) -> tuple[DeclarationExplanation, tuple[ScopeDiagnostic, ...]]: ... -def scope_browse( +def browse_scope_folders( + *, lookup: ScopeLookup, - request: tuple[str, str | PurePath | ResourceIdentity | None, bool], - /, + folder: str = ".", + target: str | PurePath | ResourceIdentity | None = None, + target_is_prospective: bool = False, ) -> ScopeBrowseResult: ... -def scope_list( +def list_scope_declarations( + *, lookup: ScopeLookup, - request: tuple[str, ScopeReportFilters | None, str | PurePath | ResourceIdentity | None, bool], - /, + folder: str, + filters: ScopeReportFilters | None = None, + target: str | PurePath | ResourceIdentity | None = None, + target_is_prospective: bool = False, ) -> ScopeListResult: ... -def scope_move_preview( - lookup: ScopeLookup, resource: str | ResourceIdentity, destination: str | PurePath +def preview_scope_move( + *, lookup: ScopeLookup, resource: str | ResourceIdentity, destination: str | PurePath ) -> tuple[MovePreview | None, tuple[ScopeDiagnostic, ...]]: ... -def serialize_scope_report(report: ScopeReport | ScopeBrowseResult | ScopeListResult) -> str: ... +def serialize_scope_report(*, report: ScopeReport | ScopeBrowseResult | ScopeListResult) -> str: ... def run_scope_command( *, request: ScopeCommandRequest, @@ -2462,7 +2562,7 @@ def render_scope_result( request: ScopeCommandRequest, use_color: bool = False, ) -> str: ... -def scope_metadata_projection(index: ScopeIndex) -> dict[str, JsonValue]: ... +def scope_metadata_projection(*, index: ScopeIndex) -> dict[str, JsonValue]: ... def effective_config_target_name( project_config: object, local_config: object, selected: str | None, / ) -> tuple[str | None, str | None]: ... @@ -2479,15 +2579,73 @@ def effective_config_settings( settings: tuple[object, object], overrides: frozenset[str], names: list[str], / ) -> dict[str, object]: ... def yaml_entry_line_index(contents: str, /) -> list[tuple[str, int]]: ... - -class NamePattern: - def __init__(self, pattern: str) -> None: ... - def matches(self, column_name: str) -> bool: ... - +def resolve_target_name( + *, project_config: ProjectConfig, local_config: LocalConfig, selected_target: str | None +) -> str | None: ... +def resolve_target_config( + *, project_config: ProjectConfig, local_config: LocalConfig, target_name: str +) -> TargetConfig: ... +def resolve_effective_adapter_name( + *, project_config: ProjectConfig, local_config: LocalConfig +) -> str: ... +def resolve_effective_collection_rendering( + *, project_config: ProjectConfig, declaration_override: CollectionRendering | None +) -> CollectionRendering: ... +def resolve_effective_scenario_config( + *, project_config: ProjectConfig, local_config: LocalConfig +) -> ScenarioConfig: ... +def scenario_local_type_overrides_for_dialect( + *, scenario_config: ScenarioConfig, sql_analysis_dialect: str | None +) -> dict[str, str]: ... +def apply_unit_test_selectors( + *, options: CompilePipelineOptions | None, discovered_inputs: DiscoveredProjectInputs +) -> CompilePipelineOptions: ... +def resolve_configured_rule_selection( + *, + options: CompilePipelineOptions, + discovered_inputs: DiscoveredProjectInputs, + project: CompiledProjectView, +) -> frozenset[CompiledObjectKey] | None: ... +def resolve_analysis_model_names( + *, + compile_inputs: CompileInputsView, + inference_profile: ExpressionInferenceProfile, + selection: CompileAnalysisSelection | None, +) -> frozenset[str] | None: ... +def compile_static_command_context( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + options: CompilePipelineOptions, + selected_keys: frozenset[CompiledObjectKey] | None = None, + relation_keys: frozenset[CompiledObjectKey] | None = None, + on_progress: Callable[[str], None] | None = None, +) -> StaticCommandContext: ... +def run_clone_pipeline_with_options( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + origin_target_name: str, + destination_target_name: str, + destination_connection: ClonePipelineConnection, + options: ClonePipelineOptions, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, +) -> ClonePipelineResult: ... +def loader_destination_parts( + *, destination: str, default_database: str | None, default_schema: str | None +) -> LoaderDestinationParts: ... +def matching_dynamic_families( + *, families: tuple[SchemaDynamicColumnFamily, ...], column_name: str +) -> tuple[SchemaDynamicColumnFamily, ...]: ... def assert_no_unresolved_sql_markers(*, sql: str, context: str) -> None: ... -def normalize_typed_sql_value( - raw_value: object, typing: tuple[str, str | None], limits: tuple[int, int, int], / -) -> object: ... +def normalize_sql_value( + *, + raw_value: object, + context: str, + explicit_type: str | None = None, + limits: SqlValueLimits | None = None, +) -> SqlValue: ... +def render_state_sql_literal(*, value: object | None, declared_type: StateSqlValueType) -> str: ... def model_header_declarations( header: tuple[object | None, object | None], model_name: str, relative_path: Path, / ) -> tuple[tuple[object, ...], tuple[object, ...]]: ... @@ -2562,8 +2720,6 @@ def select_latest_fingerprints( node_names: tuple[str, ...], filtered_node_types: tuple[str, ...] = (), ) -> FingerprintSet: ... -def lone_surrogate_site(raw_values: dict[str, object], /) -> tuple[str, str, int] | None: ... -def environment_template_names(values: list[str | None], /) -> list[str]: ... def find_python_string_locations( *, project_dir: Path, @@ -2572,9 +2728,10 @@ def find_python_string_locations( reason: str, context: str | None = None, ) -> tuple[ManualLocation, ...]: ... -def resource_identity_suggestion( - name: str, private_identity: bool, / -) -> tuple[str, str] | None: ... +def validate_resource_identity( + *, name: str, kind: str, path: Path, private_identity: bool = False +) -> None: ... +def function_node_type(*, return_columns: tuple[object, ...]) -> str: ... def load_project_config_file(project_dir: Path, /) -> ProjectConfig: ... def load_local_config_file(project_dir: Path, /) -> LocalConfig: ... def resolve_config_file_path(project_dir: Path, local: bool, /) -> Path: ... @@ -2582,7 +2739,7 @@ def validate_local_sql_analysis(project_dir: Path, flags: tuple[bool, bool, bool def sql_test_cte_span( contents: str, sql_body: str, cte_name: str, call: str | None, / ) -> tuple[int, int, int, int]: ... -def scope_normalize_path(path: str, /) -> tuple[str | None, str | None]: ... +def normalize_scope_path(*, path: str | PurePath) -> str: ... def scope_index_fingerprint(*, project_dir: Path) -> str | None: ... def load_or_build_scope_index(*, project_dir: Path, no_cache: bool = False) -> ScopeIndex: ... def model_schema_scope_usages( @@ -2607,20 +2764,19 @@ def parse_schema_columns( def column_migration_problem( model_name: str, values: dict[str, object], columns: list[tuple[str, str | None]] ) -> NativeConfigError | None: ... -def expand_declaration_references( - source: tuple[str, list[tuple[int, str, str | None, int, int]], int | None], - context: object, - consumer: tuple[object | None, bool, str], - renderers: tuple[object, dict[str, object]], -) -> tuple[str, list[tuple[int, int, int, int]], list[Any]]: ... -def expand_config_templates( +def expand_template_data( + *, value: object, - sources: tuple[dict[str, object], object, dict[str, str | None]], - flags: tuple[bool, bool, bool, str], -) -> tuple[object, list[tuple[str, str]]]: ... -def expand_effective_vars( - raw_values: dict[str, object], environment: object -) -> tuple[object, list[tuple[str, str]]]: ... + variables: dict[str, object], + context_values: dict[str, str | None], + context_label: str, + allow_context: bool, + preserve_context_tokens: bool, + preserve_unknown_context: bool, +) -> object: ... +def expand_effective_vars(raw_values: dict[str, object]) -> dict[str, object]: ... +def record_template_reads(reads: Sequence[tuple[str, str]]) -> None: ... +def resolve_run_id(*, selected_run_id: str | None) -> str: ... class NativeModelConfigBuilder: def __init__( @@ -2690,37 +2846,22 @@ def parse_cli_arguments( argv: list[str], defaults: list[tuple[str, object]], *, use_color: bool, columns: int ) -> tuple[list[tuple[str, object]] | None, int | None]: ... def run_cli(argv: Sequence[str] | None = None, handlers: object | None = None) -> int: ... +def extract_sql_test_ctes( + *, sql: str, file_label: str, syntax: SqlLexicalSyntax +) -> tuple[CompileSqlTestCte, ...]: ... def scope_test_ctes( texts: list[tuple[str, str]], syntax: dict[str, object] ) -> list[tuple[str | None, list[tuple[str, str]]]]: ... def omitted_ceremonial_select(sql: str, syntax: dict[str, object]) -> int | None: ... def extract_sql_tests_json(request_json: str) -> str: ... def validated_model_cursor_intrinsics( - sql: str, - reserved_markers: list[str], - model: tuple[str, object, object], -) -> tuple[str | None, str | None]: ... -def replace_cursor_intrinsics( - sql: str, context: str, replacements: tuple[str, str] -) -> tuple[str, bool, str | None]: ... + *, sql: str, config_values: Mapping[str, object], model_name: str +) -> str: ... +def cursor_intrinsics_analysis_sql(*, sql: str, cursor_type: object) -> str: ... +def resolve_cursor_intrinsics( + *, sql: str, start_sql: str | None = None, end_sql: str | None = None +) -> tuple[str, bool]: ... def parse_macro_arguments(text: str, nested: list[tuple[int, int]]) -> tuple[object, ...]: ... -def scan_sql_declaration_references( - sqls: list[str], -) -> list[tuple[list[tuple[int, str, str | None, int, int]], int | None]]: ... -def scan_sql_declaration_usages( - sqls: list[str], -) -> list[tuple[list[tuple[int, str, str | None, int, int]], int | None]]: ... -def interpolate_sql_batch( - sqls: list[tuple[str, str]], - sources: tuple[ - Mapping[str, object], - Mapping[str, str], - Mapping[str, str | None] | None, - Callable[..., str], - ], -) -> list[ - tuple[str | None, list[tuple[int, int, int, int]], list[tuple[str, str]], str | None] -]: ... def scan_macro_call_sites( sql: str, ) -> tuple[ @@ -2745,7 +2886,58 @@ def substitute_placeholder_defaults(sql: str, placeholders: list[tuple[str, str] def fixture_projection_facts( sql: str, dialect: str | None, / ) -> tuple[list[tuple[str, str]], list[str], list[str]]: ... +def build_python_sql_run_lifecycle( + *, selection: PythonSqlRunSelection, python_graph: PythonNodeGraph +) -> PythonSqlRunLifecyclePlan: ... +def map_expanded_offset( + *, offset: int, passes: Sequence[Sequence[ExpansionSpan]] +) -> MappedOffset: ... +def infer_fixture_column_facts( + *, query_sql: str, inference_profile: ExpressionInferenceProfile +) -> FixtureColumnInference | None: ... def find_macro_call_names(sql: str, /) -> tuple[str, ...]: ... +def build_native_declaration_contexts( + *, + lookup: ScopeLookup, + declarations: Mapping[DeclarationIdentity, EnumDeclaration | ConstantDeclaration | LoadedMacro], +) -> NativeDeclarationContexts: ... +def render_constant_declaration( + *, + declaration: ConstantDeclaration, + value_renderer: TypedSqlValueRenderer, + collection_rendering: CollectionRendering, + file_path: Path | None = None, +) -> str: ... +def render_enum_member_value(*, value: str | int) -> str: ... +def resolve_declaration_context( + *, + resolver: DeclarationScopeResolver, + file_path: Path, + resource: ResourceIdentity | None = None, +) -> DeclarationResolutionContext: ... +def substitute_sql_vars_with_spans( + *, + sql: str, + file_path: Path, + effective_vars: dict[str, object], + context_values: Mapping[str, str | None] | None = None, +) -> tuple[str, tuple[ExpansionSpan, ...]]: ... +def expand_authored_sql_with_spans( + *, + sql: str, + file_path: Path, + effective_vars: dict[str, object], + loaded_macros: dict[str, LoadedMacro], + macro_context: MacroContext, + value_renderer: TypedSqlValueRenderer, + collection_rendering: CollectionRendering, + context_values: Mapping[str, str | None] | None = None, + declarations: DeclarationResolutionContext | None = None, + declaration_resolver: DeclarationScopeResolver | None = None, +) -> tuple[str, tuple[tuple[ExpansionSpan, ...], ...]]: ... +def expand_sql_with_spans( + *, sql: str, file_path: Path, context: SqlExpansionContext +) -> tuple[str, tuple[tuple[ExpansionSpan, ...], ...]]: ... def expand_sql_macros( *, sql: str, @@ -3125,6 +3317,7 @@ def serialize_hook_entries( def named_declaration_role_kind(*, relative_path: PurePath) -> DeclarationKind | None: ... def discover_model_description_inputs(*, project_dir: Path) -> DiscoveredProjectInputs: ... def microbatch_guidance() -> tuple[str, str]: ... +def sql_analysis_opt_out_help(*, model_owned: bool = True) -> str: ... def explicit_references_help(*, allowed: str) -> str: ... def run_format_command( *, @@ -3174,6 +3367,21 @@ def resolve_loader_defaults( cli_vars: dict[str, object] | None = None, ) -> tuple[str | None, str | None]: ... +class TransientStatusReporter: + def __init__( + self, *, stream: TextIO, use_color: bool = False, enabled: bool = True + ) -> None: ... + def start(self, message: str) -> None: ... + def update(self, message: str) -> None: ... + def complete(self, *, message: str, blank_line_after: bool = False) -> None: ... + def complete_styled(self, *, message: str, blank_line_after: bool = False) -> None: ... + def error(self, message: str) -> None: ... + def close(self) -> None: ... + def clear_transient_line(self) -> None: ... + def redraw_transient_line(self) -> None: ... + def report_preflight_progress(self, message: str) -> None: ... + def write_blank_line(self) -> None: ... + class LoadProgressReporter: def __init__( self, *, stream: TextIO, use_color: bool, source_order: dict[str, int], total_count: int @@ -3184,6 +3392,77 @@ class LoadProgressReporter: def clear_transient_line(self) -> None: ... def redraw_transient_line(self) -> None: ... +class BuildProgressCallbacks: + def __init__( + self, + *, + plan: PlanOutput, + use_color: bool, + verbose: bool = False, + debug: bool = False, + event_output_path: Path | None = None, + ) -> None: ... + @property + def elapsed(self) -> float: ... + def on_node_start(self, *, name: str, resource_kind: ExecutionResourceKind) -> None: ... + def on_sub_progress(self, message: str) -> None: ... + def on_scheduler_state(self, state: SchedulerState) -> None: ... + def on_statement_complete(self, telemetry: StatementExecutionTelemetry) -> None: ... + def clear_transient_line(self) -> None: ... + def redraw_transient_line(self) -> None: ... + def on_node_complete(self, node_result: object) -> None: ... + def write_execution_event(self, result: object) -> None: ... + def close(self) -> None: ... + +class AuditProgressReporter: + def __init__( + self, + *, + entries: tuple[AuditPlanEntry, ...], + worker_limit: int, + stream: TextIO, + use_color: bool, + ) -> None: ... + def set_result_callback(self, callback: Callable[[AuditExecutionResult], None]) -> None: ... + def on_item_start(self, entry: AuditPlanEntry) -> None: ... + def on_item_complete(self, result: AuditExecutionResult) -> None: ... + def on_item_physical_complete(self, result: AuditExecutionResult) -> None: ... + def on_item_error(self, entry: AuditPlanEntry) -> None: ... + def close(self) -> None: ... + def clear_transient_line(self) -> None: ... + def redraw_transient_line(self) -> None: ... + +class NestedCommandProgressCallbacks: + def __init__( + self, + *, + total: int, + label: str, + stream: TextIO, + use_color: bool, + name_width: int = 50, + concurrency: int = 1, + ) -> None: ... + def on_item_start( + self, *, group_name: str, item_name: str, canonical_resource_name: str | None = None + ) -> None: ... + def on_item_complete( + self, + *, + group_name: str, + item_name: str, + status_text: str, + detail: str = "", + error_code: str | None = None, + error_help: str | None = None, + error_message: str | None = None, + child_rows: tuple[NestedProgressChildRow, ...] = (), + canonical_resource_name: str | None = None, + canonical_resource_id: str | None = None, + ) -> None: ... + def clear_transient_line(self) -> None: ... + def redraw_transient_line(self) -> None: ... + def observe_source_freshness_json( *, adapter: object, @@ -3220,38 +3499,12 @@ def sql_syntax_error( def check_native_sql_syntax( request: tuple[str, list[tuple[str, list[tuple[str, str]]]]], / ) -> bool: ... -def assemble_project_resource_facts( - request: tuple[ - str, - tuple[str | None, str | None, str | None] | None, - tuple[str | None, str | None, str | None, str | None], - list[tuple[str, str, str, tuple[str, str] | None]], - list[tuple[str, str | None]], - list[ - tuple[ - str, - str, - list[str], - list[tuple[str, str, str | None]], - list[tuple[str, list[tuple[str, str]]]], - ] - ], - list[tuple[str, bool, str | None, str | None]], - list[tuple[str, list[str], str | None, str | None]], - list[tuple[str, str, list[str], list[tuple[str, str, str | None]]]], - list[tuple[list[tuple[str, str, str | None]], tuple[str, str] | None]], - ], - /, -) -> tuple[ - list[list[tuple[str, str]]], - list[tuple[str | None, str | None] | None], - list[tuple[str | None, str | None, str | None, str | None]], - list[list[tuple[str, str]]], - list[list[tuple[str, str]]], - list[tuple[str, str]], - list[bool], - tuple[str, int, str, str, str | None, int | None] | None, -]: ... +def assemble_native_project_resources( + *, + inputs: CompileInputsView, + dialect: str | None, + syntax_checks: tuple[tuple[tuple[str, dict[str, str] | None], ...], ...], +) -> NativeProjectResources: ... def explicit_reference_report( project: Any, settings: dict[str, object], @@ -3599,6 +3852,143 @@ def cursor_bound_display( *, value: str, cursor_type: str | None, cursor_grain: str | None ) -> str: ... def model_materialization_label(entry: ModelPlanEntry) -> str: ... +def detect_changes( + *, + project: CompiledProjectView, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + full_refresh: bool, + expected_version_hashes: dict[str, str] | None = None, + expected_metadata_jsons: dict[str, str] | None = None, +) -> PlannerChangeResults: ... +def detect_model_changes_in_scope( + *, + project: CompiledProjectView, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + query_change_tracking: bool | None = None, + expected_version_hashes: dict[str, str] | None = None, + expected_metadata_jsons: dict[str, str] | None = None, +) -> dict[str, ChangeDetectionResult]: ... +def detect_model_changes( + *, + model: CompiledModelView, + snapshot: WarehouseSnapshot, + sql_analysis_enabled: bool, + query_change_tracking: bool, + full_refresh: bool, + function_local_hashes: dict[str, str] | None = None, + hook_version_hashes: dict[str, str] | None = None, + expected_version_hash: str | None = None, + expected_metadata_json: str | None = None, +) -> ChangeDetectionResult: ... +def resolve_model_actions( + *, scope: PlannerScope, changes: PlannerChangeResults +) -> PlannerResolvedActions: ... +def plan_reference_renames( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + state: MigrationStateInspection, + schemas: set[str], + planned: dict[str, str], +) -> tuple[ReferenceRename, ...]: ... +def replaces_incremental_relation( + *, materialization_type: MaterializationType, action: PlanAction +) -> bool: ... +def version_identity_metadata_payload(metadata_json: str | None) -> Any: ... +def get_materialization_type(model: CompiledModelView) -> MaterializationType: ... +def resolve_model_plan_action( + *, model: CompiledModelView, change_result: ChangeDetectionResult, full_refresh: bool +) -> tuple[PlanAction, PlanReason]: ... +def resolve_schema_actions( + *, schema_findings: tuple[SchemaFinding, ...], on_schema_change: OnSchemaChange | None +) -> tuple[SchemaAction, ...]: ... +def build_model_warnings( + *, + model_name: str, + materialization_type: MaterializationType, + change_result: ChangeDetectionResult, + schema_actions: tuple[SchemaAction, ...], + on_schema_change: OnSchemaChange | None, + type_enforcement: bool, +) -> tuple[PlanWarning, ...]: ... +def resolve_attachment_kind( + *, + audit: CompiledAuditView, + downstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], +) -> tuple[AuditAttachmentKind, str | None]: ... +def reads_outside_target_lineage( + *, + audit: CompiledAuditView, + upstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], +) -> bool: ... +def resolve_effective_run_scope( + *, requested_run_scope: AuditRunScope, attached_model_materialization: str | None +) -> AuditRunScope: ... +def classify_cursor_sql_type(sql_type: str) -> CursorType | None: ... +def check_cursor_type_consistency( + *, + model_name: str, + cursor_column: str | None, + cursor_type: str | None, + warehouse_columns: tuple[ColumnInfo, ...], + sql_analysis_enabled: bool, +) -> PlanWarning | None: ... +def count_microbatches( + *, + bounds: CursorBounds, + batch_size: str, + cursor_type: str, + equal_bounds_are_batch: bool = False, +) -> int: ... +def cap_microbatch_bounds( + *, + bounds: CursorBounds, + batch_size: str, + cursor_type: str, + max_batches: int, + action: MicrobatchLimitAction, +) -> CursorBounds: ... +def microbatch_limit_warning( + *, model_name: str, max_batches: int | None, batch_count: int, action: MicrobatchLimitAction +) -> str | None: ... +def resolve_microbatch_limit_config( + *, values: Mapping[str, object] +) -> tuple[int | None, MicrobatchLimitAction | None]: ... +def resolve_lineage_reference( + *, + model: CompiledModelView, + input_name: str, + models_by_name: Mapping[str, CompiledModelView], + functions_by_name: Mapping[str, CompiledFunctionView], +) -> CompileSqlReference: ... +def validate_capped_watermark_inputs( + *, + model: CompiledModelView, + models_by_name: Mapping[str, CompiledModelView], + functions_by_name: Mapping[str, CompiledFunctionView] | None = None, +) -> None: ... +def detect_schema_changes( + *, + yml_columns: tuple[ColumnInfo, ...], + inferred_columns: tuple[InferredColumn, ...] | None, + warehouse_columns: tuple[ColumnInfo, ...], + type_enforcement: bool, + inferred_schema_complete: bool, + dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] | None = None, + dialect: str | None = None, +) -> tuple[SchemaFinding, ...]: ... +def origin_reference_names( + *, + query_sql: str, + recorded: Fingerprint, + renames: tuple[ReferenceRename, ...], + dialect: str | None, +) -> dict[str, str] | None: ... +def with_declared_columns_hash(*, metadata_json: str, declared_columns_hash: str) -> str: ... +def without_declared_columns_hash(metadata_json: str) -> str: ... def model_resource_type(entry: ModelPlanEntry | None) -> str: ... def model_execution_annotation(entry: ModelPlanEntry | None) -> str: ... def materialization_type_display(resource_kind: ExecutionResourceKind) -> str: ... diff --git a/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py b/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py index c6bb91f05b..7edd187e8a 100644 --- a/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py +++ b/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py @@ -4,6 +4,7 @@ from collections.abc import Callable +from sqlbuild._native import render_state_sql_literal from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.compiler.source_freshness.constants import ( COLUMN_DATA_VERSION, @@ -21,7 +22,6 @@ SOURCE_FRESHNESS_TABLE_NAME, ) from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal def render_insert_source_freshness_records_sql( diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py index b48b7f91a2..e5d80c7ce0 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py @@ -6,7 +6,12 @@ from pathlib import Path from typing import Any -from sqlbuild._native import build_compiled_project, enforce_model_execution_limit +from sqlbuild._native import ( + build_compiled_project, + enforce_model_execution_limit, + resolve_target_config, + run_clone_pipeline_with_options, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, @@ -30,7 +35,6 @@ CompiledResourceType, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.clone_with_options import run_clone_pipeline_with_options from sqlbuild.compiler.pipeline.main.unit_test_selection import split_unit_test_selectors from sqlbuild.compiler.pipeline.models import ( ClonePipelineConnection, @@ -61,7 +65,6 @@ CloneSourceEntries, ) from sqlbuild.executor.clone.types import CloneStatus -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.types import RetentionDecreasePolicy diff --git a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py index edc2843934..8132be96ca 100644 --- a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py +++ b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py @@ -7,15 +7,13 @@ from typing import TextIO from sqlbuild._native import ( + BuildProgressCallbacks, ConnectionProgressReporter, enforce_plan_safety_policies, + format_build_footer, format_execution_header, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands.classes.build_progress_callbacks import ( - BuildProgressCallbacks, - format_build_footer, -) from sqlbuild.cli.commands.constants import ( EXECUTION_LIMIT_DBT_NOTE, MISSING_ORIGIN_DBT_HELP, diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py index 531dfe13fe..2a0a9f7fc9 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py @@ -5,6 +5,7 @@ import sys import time +from sqlbuild._native import TransientStatusReporter from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import ( JanitorCompileContext, @@ -15,7 +16,6 @@ ) from sqlbuild.executor.janitor.main.plan import build_janitor_plan from sqlbuild.executor.janitor.models import JanitorDirectModeSettings, JanitorPlan -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py index 5a1a66cd8e..15aebce087 100644 --- a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py @@ -6,7 +6,11 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import discover_project_inputs, validate_preview_target +from sqlbuild._native import ( + build_effective_settings_config, + discover_project_inputs, + validate_preview_target, +) from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( resolve_adapter_connection_context, ) @@ -19,7 +23,6 @@ build_command_progress_reporters, ) from sqlbuild.cli.progress.models import CommandProgressReporters -from sqlbuild.compiler.compile.main.effective_settings import build_effective_settings_config from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.presentation.main.supports_color import supports_color diff --git a/src/sqlbuild/cli/commands/_helpers/rules/command.py b/src/sqlbuild/cli/commands/_helpers/rules/command.py index b22b1aca8c..d158cfa71a 100644 --- a/src/sqlbuild/cli/commands/_helpers/rules/command.py +++ b/src/sqlbuild/cli/commands/_helpers/rules/command.py @@ -7,7 +7,11 @@ from dataclasses import replace from pathlib import Path -from sqlbuild._native import build_project_graph, discover_project_inputs +from sqlbuild._native import ( + build_project_graph, + discover_project_inputs, + resolve_effective_adapter_name, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands.constants import ( @@ -29,9 +33,6 @@ from sqlbuild.rule_engine.main.run_rules import run_rules from sqlbuild.rule_engine.main.skills import install_skills from sqlbuild.rule_engine.models import Rule, RulesConfig, RulesResult, RulesRunResult -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) def run_rules_command(request: RulesCommandRequest) -> int: diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py b/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py index ccb309ab74..155eaae005 100644 --- a/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py +++ b/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py @@ -4,14 +4,11 @@ from pathlib import Path -from sqlbuild._native import resolve_project_connection_config +from sqlbuild._native import resolve_effective_adapter_name, resolve_project_connection_config from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands.models import AdapterConnectionContext from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) def resolve_adapter_connection_context( diff --git a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py deleted file mode 100644 index 48a1ea6a9e..0000000000 --- a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py +++ /dev/null @@ -1,1580 +0,0 @@ -"""Build progress output callbacks and summary formatting.""" - -from __future__ import annotations - -import logging -import sys -import threading -import time -from dataclasses import replace -from pathlib import Path - -from sqlbuild._native import ( - ExecutionEventWriter, - NativeProgressProjector, - current_native_progress_projector, - cursor_bound_display, - format_completion_line, - format_expectation_detail, - format_expectation_name, - format_no_input_rows, - format_parameterized_test_label, - format_status_cell, - format_summary_footer, - format_waiting_on_empty_inputs, - inclusive_cursor_end, - materialization_type_display, - model_execution_annotation, - model_resource_type, -) -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.progress.models import AuditDisplayEntry, ExecutionCounts -from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditOutcome, AuditRunScope -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput -from sqlbuild.compiler.python_nodes.types import PythonNodeStatus -from sqlbuild.cost.models import StatementExecutionTelemetry -from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.executor.build.models import ( - BuildExecutionResult, - FunctionExecutionResult, - SchedulerState, - SeedExecutionResult, -) -from sqlbuild.executor.build.types import BuildStatus, ExecutionStatus -from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult -from sqlbuild.executor.run.models import HookExecutionResult, ModelExecutionResult -from sqlbuild.executor.run.types import HookPhase -from sqlbuild.executor.testing.main.resource_id import sql_test_resource_id -from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult -from sqlbuild.executor.testing.types import SqlTestOutcome -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator -from sqlbuild.presentation.main.coded_error_text import format_coded_error -from sqlbuild.presentation.main.inline_error_lines import format_inline_error_lines -from sqlbuild.presentation.main.terminal_columns import terminal_columns -from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator -from sqlbuild.presentation.main.tree_connector import tree_connector -from sqlbuild.presentation.types import CompletionState -from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry - -_TYPE_WIDTH: int = 10 -_MIN_NAME_WIDTH: int = 20 -_NAME_PADDING: int = 2 -_SUB_INDENT: int = 2 -_TREE_CHILD_NAME_INDENT: int = _SUB_INDENT + 4 -_HOOK_PHASE_WIDTH: int = 10 -_HOOK_TYPE_WIDTH: int = 8 -_HOOK_MIN_LABEL_WIDTH: int = 24 -_HOOK_MAX_LABEL_WIDTH: int = 56 -_SPINNER_TICK_SECONDS: float = 0.1 -_SKIPPED_ROLLUP_CAP: int = 20 -_ROW_COUNT_BILLION: int = 1_000_000_000 -_ROW_COUNT_MILLION: int = 1_000_000 -_ROW_COUNT_THOUSAND: int = 1_000 -_ROW_COUNT_ABBREVIATION_THRESHOLD: int = 10_000 -_ACTIVE_SPINNER_FRAMES: tuple[str, ...] = ( - "⠋", - "⠙", - "⠹", - "⠸", - "⠼", - "⠴", - "⠦", - "⠧", - "⠇", - "⠏", -) -_RUNTIME_DIAGNOSTICS_LOGGER: logging.Logger = logging.getLogger("sqlbuild.executor.runtime") -_SCHEDULER_DIAGNOSTIC_LIMIT: int = 50 -_QUERY_DIAGNOSTIC_LIMIT: int = 25 - - -def _result_resource_name(result: object) -> str | None: - if isinstance(result, ModelExecutionResult): - return result.model_name - if isinstance(result, SeedExecutionResult): - return result.seed_name - if isinstance(result, FunctionExecutionResult): - return result.function_name - if isinstance(result, LoadExecutionResult): - return result.source_name - if isinstance(result, PythonNodeExecutionResult): - return result.node_name - if isinstance(result, SqlTestExecutionResult): - return result.test_name - return None - - -def _result_resource_id(result: object) -> str | None: - if isinstance(result, ModelExecutionResult): - return f"model:{result.model_name}" - if isinstance(result, SeedExecutionResult): - return f"seed:{result.seed_name}" - if isinstance(result, FunctionExecutionResult): - return f"{result.function_kind}:{result.function_name}" - if isinstance(result, LoadExecutionResult): - return f"source:{result.source_name}" - if isinstance(result, PythonNodeExecutionResult): - return f"{result.kind.value}:{result.node_name}" - if isinstance(result, SqlTestExecutionResult): - return sql_test_resource_id( - test_name=result.test_name, - source_path=result.source_path, - block_index=result.block_index, - case_name=result.case_name, - ) - return None - - -def _with_canonical_duration(*, result: object, duration_ms: float) -> object: - if isinstance(result, ModelExecutionResult): - return replace(result, duration_ms=int(duration_ms)) - if isinstance(result, SeedExecutionResult): - return replace(result, duration_ms=int(duration_ms)) - if isinstance(result, FunctionExecutionResult): - return replace(result, duration_ms=int(duration_ms)) - if isinstance(result, LoadExecutionResult): - return replace(result, duration_ms=int(duration_ms)) - return result - - -class BuildProgressCallbacks: - """Encapsulates live build progress output state and callbacks.""" - - def __init__( - self, - *, - plan: PlanOutput, - use_color: bool, - verbose: bool = False, - debug: bool = False, - event_output_path: Path | None = None, - ) -> None: - self._plan: PlanOutput = plan - self._model_entry_map: dict[str, ModelPlanEntry] = { - entry.name: entry for entry in plan.model_entries - } - self._test_results_by_model: dict[str, list[SqlTestExecutionResult]] = {} - self._total: int = ( - len(plan.model_entries) - + len(plan.seed_entries) - + len(plan.function_entries) - + sum( - 1 - for key in plan.execution_order - if key.resource_type == CompiledResourceType.SOURCE - and plan.source_map.get(key.name) is not None - and plan.source_map[key.name].loader is not None - ) - ) - self._counter: int = 0 - self._use_color: bool = use_color - self._style: CliStyle = CliStyle(use_color=self._use_color) - self._verbose: bool = verbose - self._debug: bool = debug - self._is_tty: bool = hasattr(sys.stdout, "isatty") and sys.stdout.isatty() and not debug - self._stream = sys.stderr if debug else sys.stdout - self._lines: TransientLineCoordinator = shared_transient_line_coordinator() - self._start_time: float = time.monotonic() - self._current_node_name: str = "" - self._current_node_type: ExecutionResourceKind = ExecutionResourceKind.TABLE - self._current_sub_message: str = "" - self._spinner_frame_index: int = 0 - self._spinner_line_active: bool = False - self._runtime_diagnostic_lock: threading.Lock = threading.Lock() - self._spinner_stop_event: threading.Event | None = None - self._spinner_thread: threading.Thread | None = None - self._cursor_hidden: bool = False - self._event_writer: ExecutionEventWriter = ExecutionEventWriter(path=event_output_path) - self._scheduler_diagnostic_count: int = 0 - self._scheduler_diagnostic_omitted: int = 0 - self._last_omitted_scheduler_state: SchedulerState | None = None - self._query_diagnostic_count: int = 0 - self._query_diagnostic_omitted: int = 0 - self._last_omitted_query: StatementExecutionTelemetry | None = None - self._projector: NativeProgressProjector | None = current_native_progress_projector() - if self._projector is not None: - display_names: list[str] = [] - for key in plan.execution_order: - if key.resource_type == CompiledResourceType.SQL_TEST: - continue - if key.resource_type == CompiledResourceType.SOURCE: - source: SourceEntry | None = plan.source_map.get(key.name) - if source is None or source.loader is None: - continue - display_names.append(key.name) - self._projector.configure_resources( - ordinals={name: index for index, name in enumerate(display_names, start=1)}, - total=self._total, - ) - for test_entry in plan.test_entries: - self._projector.expect_resource_enrichment(resource_name=test_entry.name) - - ctr_width: int = len(str(self._total)) * 2 + 1 - self._prefix_width: int = 2 + ctr_width + 2 - - max_name_len: int = 0 - entry: ModelPlanEntry - for entry in plan.model_entries: - annotation: str = model_execution_annotation(entry) - display_name: str = entry.name - if annotation: - display_name = f"{entry.name} ({annotation})" - max_name_len = max(max_name_len, len(display_name)) - seed_entry: object - for seed_entry in plan.seed_entries: - max_name_len = max(max_name_len, len(getattr(seed_entry, "name", str(seed_entry)))) - function_entry: object - for function_entry in plan.function_entries: - max_name_len = max( - max_name_len, len(getattr(function_entry, "name", str(function_entry))) - ) - test_entry: object - for test_entry in plan.test_entries: - max_name_len = max(max_name_len, len(getattr(test_entry, "name", str(test_entry)))) - chain_step: object - for chain_step in getattr(test_entry, "chain", ()): - if getattr(chain_step, "expected_cte_sql", None): - max_name_len = max( - max_name_len, - _tree_child_name_length( - format_expectation_name(str(getattr(chain_step, "model_name", ""))) - ), - ) - assertion: object - for assertion in getattr(test_entry, "assertions", ()): - max_name_len = max( - max_name_len, - _tree_child_name_length( - format_expectation_name(f"assertion {getattr(assertion, 'name', '')}") - ), - ) - self._name_width: int = max(max_name_len + _NAME_PADDING, _MIN_NAME_WIDTH) - - @property - def elapsed(self) -> float: - return time.monotonic() - self._start_time - - def _write_sql_block(self, sql: str) -> None: - """Write a SQL block with minimal indent and dim styling.""" - - self._stream.write("\n") - sql_line: str - for sql_line in _format_display_sql(sql).split("\n"): - styled: str = self._style.muted(f" {sql_line}") - self._stream.write(f"{styled}\n") - self._stream.write("\n") - - def _write_log_block(self, message: str) -> None: - """Write a log message with indent and muted styling.""" - - lines: list[str] = message.splitlines() or [""] - prefix: str = self._style.log_label(" log ") - first_content: str = self._style.muted(lines[0]) - styled_first: str = f"{prefix}{first_content}" - self._stream.write(f"\n{styled_first}\n") - line: str - for line in lines[1:]: - continuation: str = f" {line}" - styled_continuation: str = self._style.muted(continuation) - self._stream.write(f"{styled_continuation}\n") - - def on_node_start(self, *, name: str, resource_kind: ExecutionResourceKind) -> None: - if self._projector is not None: - self._projector.expect_resource_enrichment(resource_name=name) - return - self._current_node_name = name - self._current_node_type = resource_kind - self._current_sub_message = "" - if self._is_tty: - self._hide_cursor() - with self._lines.lock: - self._spinner_line_active = True - self._lines.claim(stream=self._stream, owner=self) - self._write_spinner_line() - self._start_spinner_loop() - - def on_sub_progress(self, message: str) -> None: - self._current_sub_message = message - if self._is_tty: - self._write_spinner_line() - - def on_scheduler_state(self, state: SchedulerState) -> None: - """Report one deduplicated concurrent scheduler state transition.""" - - with self._runtime_diagnostic_lock: - self._scheduler_diagnostic_count += 1 - if self._scheduler_diagnostic_count > _SCHEDULER_DIAGNOSTIC_LIMIT: - self._scheduler_diagnostic_omitted += 1 - self._last_omitted_scheduler_state = state - return - self._write_runtime_diagnostic(_scheduler_diagnostic_message(state)) - - def on_statement_complete(self, telemetry: StatementExecutionTelemetry) -> None: - """Report a bounded sample of completed Snowflake statement identities.""" - - with self._runtime_diagnostic_lock: - self._query_diagnostic_count += 1 - if self._query_diagnostic_count > _QUERY_DIAGNOSTIC_LIMIT: - self._query_diagnostic_omitted += 1 - self._last_omitted_query = telemetry - return - self._write_runtime_diagnostic(_query_diagnostic_message(telemetry)) - - def _write_runtime_diagnostic(self, message: str) -> None: - _RUNTIME_DIAGNOSTICS_LOGGER.debug(message) - if self._debug: - return - self._lines.write_persistent(stream=self._stream, text=f"{message}\n") - - def _write_runtime_diagnostic_summaries(self) -> None: - with self._runtime_diagnostic_lock: - scheduler_omitted: int = self._scheduler_diagnostic_omitted - last_scheduler_state: SchedulerState | None = self._last_omitted_scheduler_state - query_omitted: int = self._query_diagnostic_omitted - last_query: StatementExecutionTelemetry | None = self._last_omitted_query - if scheduler_omitted and last_scheduler_state is not None: - self._write_runtime_diagnostic( - f"Scheduler diagnostics {scheduler_omitted} transitions omitted " - f"after {_SCHEDULER_DIAGNOSTIC_LIMIT}; final state: " - f"{_scheduler_diagnostic_message(last_scheduler_state)}" - ) - if query_omitted and last_query is not None: - self._write_runtime_diagnostic( - f"Query diagnostics {query_omitted} statements omitted after " - f"{_QUERY_DIAGNOSTIC_LIMIT}; final query: " - f"{_query_diagnostic_message(last_query)}" - ) - - def clear_transient_line(self) -> None: - """Erase the live spinner row so a persistent line can take its place.""" - - with self._lines.lock: - self._stream.write("\r\033[K") - self._stream.flush() - - def redraw_transient_line(self) -> None: - """Draw the live spinner row again below persistent output.""" - - self._write_spinner_line() - - def _write_spinner_line(self) -> None: - with self._lines.lock: - if self._spinner_line_active: - self._draw_spinner_line() - - def _draw_spinner_line(self) -> None: - ctr: str = f"{self._counter + 1}/{self._total}".rjust(len(str(self._total)) * 2 + 1) - display_type: str = materialization_type_display(self._current_node_type) - status: str = self._style.status(status=_ACTIVE_SPINNER_FRAMES[self._spinner_frame_index]) - self._spinner_frame_index = (self._spinner_frame_index + 1) % len(_ACTIVE_SPINNER_FRAMES) - name_display: str = _truncate_name(name=self._current_node_name, width=self._name_width) - if self._current_sub_message: - name_display = f"{name_display} {self._current_sub_message}" - nw: int = self._name_width - plain_part: str = f" {ctr} {display_type:<{_TYPE_WIDTH}}{name_display:<{nw}} " - max_plain_width: int = max(terminal_columns() - 2, _MIN_NAME_WIDTH) - if len(plain_part) > max_plain_width: - plain_part = f"{plain_part[: max_plain_width - 4]}... " - self._stream.write(f"\r\033[K{plain_part}{status}") - self._stream.flush() - - def _start_spinner_loop(self) -> None: - self._stop_spinner_loop() - stop_event: threading.Event = threading.Event() - self._spinner_stop_event = stop_event - spinner_thread: threading.Thread = threading.Thread( - target=self._spin_until_stopped, - args=(stop_event,), - daemon=True, - ) - self._spinner_thread = spinner_thread - spinner_thread.start() - - def _stop_spinner_loop(self) -> None: - if self._spinner_stop_event is not None: - self._spinner_stop_event.set() - if self._spinner_thread is not None and self._spinner_thread.is_alive(): - self._spinner_thread.join(timeout=0.2) - self._spinner_stop_event = None - self._spinner_thread = None - - def _spin_until_stopped(self, stop_event: threading.Event) -> None: - while not stop_event.wait(_SPINNER_TICK_SECONDS): - self._write_spinner_line() - - def _hide_cursor(self) -> None: - if self._cursor_hidden: - return - with self._lines.lock: - self._stream.write("\033[?25l") - self._stream.flush() - self._cursor_hidden = True - - def _show_cursor(self) -> None: - if not self._cursor_hidden: - return - with self._lines.lock: - self._stream.write("\033[?25h") - self._stream.flush() - self._cursor_hidden = False - - def on_node_complete(self, node_result: object) -> None: - if self._projector is not None: - resource_name: str | None = _result_resource_name(node_result) - if resource_name is not None: - duration_ms: float | None = self._projector.consume_resource_terminal( - resource_name=resource_name, - resource_id=_result_resource_id(node_result), - ) - if duration_ms is None: - return - node_result = _with_canonical_duration(result=node_result, duration_ms=duration_ms) - try: - self._write_node_result(node_result) - finally: - self._write_execution_event(node_result) - - def _write_node_result(self, node_result: object) -> None: - if isinstance(node_result, SqlTestExecutionResult): - if node_result.outcome != SqlTestOutcome.PASS: - self._write_failed_test_result(test_result=node_result) - return - step: object - for step in node_result.step_results: - if hasattr(step, "model_name"): - self._test_results_by_model.setdefault(step.model_name, []).append(node_result) - return - - self._stop_spinner_loop() - if self._is_tty: - with self._lines.lock: - self._spinner_line_active = False - self._lines.release(owner=self) - self._stream.write("\r\033[K") - self._stream.flush() - self._show_cursor() - - self._counter += 1 - ctr: str = f"{self._counter}/{self._total}".rjust(len(str(self._total)) * 2 + 1) - - if isinstance(node_result, SeedExecutionResult): - status: str = _execution_status_display(node_result.status) - duration: str = _format_duration(node_result.duration_ms) - seed_name: str = _truncate_name(name=node_result.seed_name, width=self._name_width) - self._write_top_level_result_line( - ctr=ctr, - resource_type="seed", - name=seed_name, - status=status, - duration=duration, - ) - if node_result.status == ExecutionStatus.FAILED and node_result.error_message: - self._write_error_detail( - error_code=node_result.error_code, - error_message=node_result.error_message, - error_help=node_result.error_help, - ) - self._stream.flush() - return - - if isinstance(node_result, FunctionExecutionResult): - status: str = _execution_status_display(node_result.status) - duration: str = _format_duration(node_result.duration_ms) - function_name: str = _truncate_name( - name=node_result.function_name, width=self._name_width - ) - self._write_top_level_result_line( - ctr=ctr, - resource_type=node_result.function_kind, - name=function_name, - status=status, - duration=duration, - ) - if node_result.status == ExecutionStatus.FAILED and node_result.error_message: - self._write_error_detail( - error_code=node_result.error_code, - error_message=node_result.error_message, - error_help=node_result.error_help, - ) - self._stream.flush() - return - - if isinstance(node_result, LoadExecutionResult): - status = _execution_status_display(node_result.status) - duration = _format_duration(node_result.duration_ms) - source_name: str = _truncate_name(name=node_result.source_name, width=self._name_width) - detail: str = _load_result_detail(node_result) - self._write_top_level_result_line( - ctr=ctr, - resource_type=node_result.resource_kind.value, - name=source_name, - status=status, - duration=duration, - detail=detail, - ) - event: LifeCycleEvent - for event in node_result.lifecycle_events: - if event.kind == LifeCycleEventKind.LOG: - self._write_log_block(event.content) - elif self._verbose and event.kind == LifeCycleEventKind.SQL: - self._write_sql_block(event.content) - if node_result.status == ExecutionStatus.FAILED and node_result.error_message: - self._write_error_detail( - error_code=None, - error_message=node_result.error_message, - error_help=None, - ) - self._stream.flush() - return - - if isinstance(node_result, ModelExecutionResult): - self._write_model_result(ctr=ctr, model_result=node_result) - - def _write_execution_event(self, node_result: object) -> None: - self._event_writer.write_build_result( - result=node_result, - plan=self._plan, - ) - if isinstance(node_result, ModelExecutionResult): - for audit_result in node_result.audit_results: - self._event_writer.write_build_result( - result=audit_result, - plan=self._plan, - ) - - def write_execution_event(self, result: object) -> None: - """Write one result to the shared structured event channel.""" - - self._write_execution_event(result) - - def close(self) -> None: - """Close the optional structured execution event stream.""" - - self._write_runtime_diagnostic_summaries() - self._event_writer.close() - - def _write_model_result(self, *, ctr: str, model_result: ModelExecutionResult) -> None: - plan_entry: ModelPlanEntry | None = self._model_entry_map.get(model_result.model_name) - display_type: str = model_resource_type(plan_entry) - annotation: str = model_execution_annotation(plan_entry) - name_display: str = model_result.model_name - if annotation: - name_display = f"{model_result.model_name} ({annotation})" - name_display = _truncate_name(name=name_display, width=self._name_width) - - status: str = _execution_status_display(model_result.status) - duration: str = _format_duration(model_result.duration_ms) - detail: str = "" - if model_result.status == ExecutionStatus.FAILED and model_result.failed_phase is not None: - detail = f" {model_result.failed_phase}" - elif model_result.status == ExecutionStatus.SKIPPED: - duration = "" - detail = _model_skip_detail(model_result) - - self._write_top_level_result_line( - ctr=ctr, - resource_type=display_type, - name=name_display, - status=status, - duration=duration, - detail=detail, - ) - if self._verbose: - event: LifeCycleEvent - for event in _resolve_verbose_events( - model_result=model_result, - plan_entry=plan_entry, - ): - if event.kind == LifeCycleEventKind.SQL: - self._write_sql_block(event.content) - elif event.kind == LifeCycleEventKind.LOG: - self._write_log_block(event.content) - - batch_line: str | None = _format_streaming_batch_summary( - model_result=model_result - ) or _format_empty_cursor_inputs(model_result=model_result) - if batch_line is not None: - self._stream.write(f"{self._style.muted(batch_line)}\n") - self._stream.flush() - if model_result.waiting_on_empty_inputs and model_result.empty_cursor_inputs: - waiting_line: str = ( - f" {format_waiting_on_empty_inputs(model_result.empty_cursor_inputs)}" - ) - self._stream.write(f"{self._style.warning(waiting_line)}\n") - self._stream.flush() - - sub_pad: str = " " * (self._prefix_width + _SUB_INDENT) - sub_nw: int = self._name_width - _SUB_INDENT - hook_label_width: int = _hook_label_width(model_result.hook_results) - - self._write_hook_results( - hook_results=model_result.hook_results, - phase=HookPhase.PRE_HOOKS, - sub_pad=sub_pad, - label_width=hook_label_width, - ) - - test_result: SqlTestExecutionResult - for test_result in self._test_results_by_model.get(model_result.model_name, []): - test_status: str = self._style.status(status=_test_outcome_display(test_result.outcome)) - test_name: str = format_parameterized_test_label( - name=test_result.test_name, - source_path=test_result.source_path, - parameter_schema=test_result.parameter_schema, - parameter_values=test_result.parameter_values, - ) - self._stream.write( - f"{sub_pad}{self._style.muted(f'{"test":<{_TYPE_WIDTH}}')}" - f"{test_name:<{sub_nw}} {test_status}\n" - ) - expectation_pad: str = f"{sub_pad} " - expectation_type_width: int = _TYPE_WIDTH - 2 - step_result: StepResult - for step_result in test_result.step_results: - expectation_status: str = self._style.status( - status=_test_outcome_display(step_result.outcome) - ) - expectation_name: str = format_expectation_name(step_result.model_name) - expectation_detail: str = format_expectation_detail(step_result) - self._stream.write( - f"{expectation_pad}{self._style.muted(f'{"expect":<{expectation_type_width}}')}" - f"{expectation_name:<{sub_nw}} {expectation_status}{expectation_detail}\n" - ) - - display_audits: list[AuditDisplayEntry] = _aggregate_audit_results( - model_result.audit_results - ) - - entry: AuditDisplayEntry - for entry in display_audits: - audit_status: str = self._style.status(status=_audit_outcome_display(entry.outcome)) - audit_name: str = _truncate_name(name=entry.display_name, width=sub_nw) - audit_detail: str = "" - if entry.reused: - audit_detail = " proof reused" - if entry.outcome != AuditOutcome.PASS and entry.total_row_count > 0: - row_label: str = "row" if entry.total_row_count == 1 else "rows" - audit_detail = f" {entry.total_row_count} {row_label}" - if entry.batch_total > 1: - audit_detail = f" {entry.batch_pass}/{entry.batch_total}" + audit_detail - audit_line: str = ( - f"{sub_pad}{self._style.muted(f'{entry.label:<{_TYPE_WIDTH}}')}" - f"{audit_name:<{sub_nw}} {audit_status}{audit_detail}\n" - ) - self._stream.write(audit_line) - - if self._verbose and entry.executed_sql is not None: - self._write_sql_block(entry.executed_sql) - - self._write_hook_results( - hook_results=model_result.hook_results, - phase=HookPhase.POST_HOOKS, - sub_pad=sub_pad, - label_width=hook_label_width, - ) - - if model_result.status == ExecutionStatus.FAILED and model_result.error_message: - self._write_error_detail( - error_code=model_result.error_code, - error_message=model_result.error_message, - error_help=model_result.error_help, - ) - - self._stream.flush() - - def _write_failed_test_result(self, *, test_result: SqlTestExecutionResult) -> None: - """Write a failing test row inline since its host model rows will not run.""" - - self._stop_spinner_loop() - if self._is_tty: - with self._lines.lock: - self._spinner_line_active = False - self._lines.release(owner=self) - self._stream.write("\r\033[K") - self._stream.flush() - self._show_cursor() - blank_ctr: str = " " * (len(str(self._total)) * 2 + 1) - test_status: str = self._style.status(status=_test_outcome_display(test_result.outcome)) - test_name: str = _truncate_name( - name=format_parameterized_test_label( - name=test_result.test_name, - source_path=test_result.source_path, - parameter_schema=test_result.parameter_schema, - parameter_values=test_result.parameter_values, - ), - width=self._name_width, - ) - self._stream.write( - f" {blank_ctr} {self._style.muted(f'{"test":<{_TYPE_WIDTH}}')}" - f"{test_name:<{self._name_width}} {test_status}\n" - ) - expectation_pad: str = " " * (self._prefix_width + _SUB_INDENT + 2) - expectation_type_width: int = _TYPE_WIDTH - 2 - tree_child_nw: int = self._name_width - _TREE_CHILD_NAME_INDENT - step_result: StepResult - for index, step_result in enumerate(test_result.step_results): - expectation_status: str = self._style.status( - status=_test_outcome_display(step_result.outcome) - ) - expectation_name: str = format_expectation_name(step_result.model_name) - expectation_detail: str = format_expectation_detail(step_result) - last: bool = ( - index == len(test_result.step_results) - 1 and not test_result.error_message - ) - connector: str = tree_connector(style=self._style, last=last) - self._stream.write( - f"{expectation_pad}{connector} " - f"{self._style.muted(f'{"expect":<{expectation_type_width}}')}" - f"{expectation_name:<{tree_child_nw}} {expectation_status}{expectation_detail}\n" - ) - if test_result.error_message: - self._write_error_detail( - error_code=test_result.error_code, - error_message=test_result.error_message, - error_help=test_result.error_help, - tree_child=True, - ) - self._stream.flush() - - def _write_hook_results( - self, - *, - hook_results: tuple[HookExecutionResult, ...], - phase: HookPhase, - sub_pad: str, - label_width: int, - ) -> None: - hook_result: HookExecutionResult - for hook_result in hook_results: - if hook_result.phase != phase: - continue - hook_status: str = self._style.status( - status=_execution_status_display(hook_result.status) - ) - hook_name: str = _truncate_name(name=hook_result.label, width=label_width) - label: str = "pre_hook" if phase == HookPhase.PRE_HOOKS else "post_hook" - detail: str = _hook_skip_detail(hook_result) - self._stream.write( - f"{sub_pad}{self._style.muted(f'{label:<{_HOOK_PHASE_WIDTH}}')}" - f"{self._style.muted(f'{hook_result.hook_type:<{_HOOK_TYPE_WIDTH}}')}" - f"{hook_name:<{label_width}} {hook_status}{detail}\n" - ) - - def _write_top_level_result_line( - self, - *, - ctr: str, - resource_type: str, - name: str, - status: str, - duration: str, - detail: str = "", - ) -> None: - nw: int = self._name_width - status_cell: str = format_status_cell(style=self._style, status=status) - rendered_duration: str = self._style.muted(duration) if duration else "" - rendered_type: str = self._style.muted(f"{resource_type:<{_TYPE_WIDTH}}") - self._stream.write( - f" {ctr} {rendered_type}{name:<{nw}} {status_cell} {rendered_duration}{detail}\n" - ) - - def _write_error_detail( - self, - *, - error_code: str | None, - error_message: str, - error_help: str | None = None, - tree_child: bool = False, - ) -> None: - child_offset: int = _SUB_INDENT + 2 if tree_child else 0 - pad: str = " " * (self._prefix_width + child_offset) - connector: str = tree_connector(style=self._style, last=True) if tree_child else "" - connector_suffix: str = f"{connector} " if tree_child else "" - label_width: int = _TYPE_WIDTH - 3 if tree_child else _TYPE_WIDTH - 1 - label_padding: str = " " * max(0, label_width - len("error")) - label: str = f"{self._style.error_muted('error')}{label_padding}" - plain_prefix_width: int = len(pad) + (4 if tree_child else 0) + label_width + 1 - content_width: int = max(terminal_columns() - plain_prefix_width, 1) - lines: list[str] = format_inline_error_lines( - error_code=error_code, - error_message=error_message, - error_help=error_help, - content_width=content_width, - style=self._style, - ) - continuation_pad: str = " " * plain_prefix_width - for line_index, line in enumerate(lines): - if line_index == 0: - self._stream.write(f"{pad}{connector_suffix}{label} {line}\n") - else: - self._stream.write(f"{continuation_pad}{line}\n") - - -def _build_audit_results(result: BuildExecutionResult) -> tuple[AuditExecutionResult, ...]: - audits: list[AuditExecutionResult] = [] - model_result: ModelExecutionResult - for model_result in result.model_results: - audits.extend(model_result.audit_results) - return (*audits, *result.source_audit_results, *result.end_audit_results) - - -def _count_build_footer_results( - *, - result: BuildExecutionResult, - python_node_results: tuple[PythonNodeExecutionResult, ...], -) -> ExecutionCounts: - pass_count: int = 0 - warn_count: int = 0 - fail_count: int = 0 - skip_count: int = 0 - - model_result: ModelExecutionResult - for model_result in result.model_results: - if model_result.status == ExecutionStatus.SUCCESS and model_result.warning_messages: - warn_count += 1 - elif model_result.status == ExecutionStatus.SUCCESS: - pass_count += 1 - elif model_result.status == ExecutionStatus.FAILED: - fail_count += 1 - elif model_result.status == ExecutionStatus.SKIPPED: - skip_count += 1 - audit_r: AuditExecutionResult - for audit_r in _build_audit_results(result): - if audit_r.outcome == AuditOutcome.PASS: - pass_count += 1 - elif audit_r.outcome == AuditOutcome.WARN: - warn_count += 1 - elif audit_r.outcome == AuditOutcome.ERROR: - fail_count += 1 - - seed_result: SeedExecutionResult - for seed_result in result.seed_results: - if seed_result.status == ExecutionStatus.SUCCESS: - pass_count += 1 - elif seed_result.status == ExecutionStatus.FAILED: - fail_count += 1 - elif seed_result.status == ExecutionStatus.SKIPPED: - skip_count += 1 - - function_result: FunctionExecutionResult - for function_result in result.function_results: - if function_result.status == ExecutionStatus.SUCCESS: - pass_count += 1 - elif function_result.status == ExecutionStatus.FAILED: - fail_count += 1 - elif function_result.status == ExecutionStatus.SKIPPED: - skip_count += 1 - warn_count += len(function_result.warning_messages) - - load_result: LoadExecutionResult - for load_result in result.load_results: - if load_result.status == ExecutionStatus.SUCCESS and load_result.warning_messages: - warn_count += 1 - elif load_result.status == ExecutionStatus.SUCCESS: - pass_count += 1 - elif load_result.status == ExecutionStatus.FAILED: - fail_count += 1 - elif load_result.status == ExecutionStatus.SKIPPED: - skip_count += 1 - - test_r: SqlTestExecutionResult - for test_r in result.test_results: - if test_r.outcome == SqlTestOutcome.PASS: - pass_count += 1 - else: - fail_count += 1 - - python_result: PythonNodeExecutionResult - for python_result in python_node_results: - if python_result.status == PythonNodeStatus.SUCCESS and python_result.warning_messages: - warn_count += 1 - elif python_result.status == PythonNodeStatus.SUCCESS: - pass_count += 1 - elif python_result.status == PythonNodeStatus.FAILED: - fail_count += 1 - elif python_result.status == PythonNodeStatus.SKIPPED: - skip_count += 1 - - return ExecutionCounts( - pass_count=pass_count, - warn_count=warn_count, - fail_count=fail_count, - skip_count=skip_count, - ) - - -def format_build_footer( - *, - result: BuildExecutionResult, - elapsed: float, - use_color: bool, - python_node_results: tuple[PythonNodeExecutionResult, ...] = (), -) -> str: - lines: list[str] = [] - style: CliStyle = CliStyle(use_color=use_color) - python_fail_count: int = sum( - 1 - for python_result in python_node_results - if python_result.status == PythonNodeStatus.FAILED - ) - - counts: ExecutionCounts = _count_build_footer_results( - result=result, python_node_results=python_node_results - ) - insufficient_count: int = result.insufficient_count - elapsed_str: str = f"{elapsed:.2f}s" - counts_summary: str = format_summary_footer( - counts=( - ("PASS", counts.pass_count), - ("WARN", counts.warn_count), - ("FAIL", counts.fail_count), - ("INSUFFICIENT", insufficient_count), - ("SKIP", counts.skip_count), - ("TOTAL", counts.total_count + insufficient_count), - ), - use_color=style.use_color, - elapsed=elapsed_str, - ) - if result.status == BuildStatus.FAILED or python_fail_count: - state: CompletionState = CompletionState.FAIL - label: str = "Completed with errors" - elif result.warning_count > 0 or counts.warn_count > 0: - state = CompletionState.WARN - label = "Completed with warnings" - else: - state = CompletionState.OK - label = "Completed successfully" - warning_lines: list[str] = _format_warning_details( - result=result, python_node_results=python_node_results, style=style - ) - if warning_lines: - lines.extend(warning_lines) - lines.append( - format_completion_line(style=style, state=state, label=label, summary=counts_summary) - ) - - failure_lines: list[str] = _format_failure_details(result=result, style=style) - failure_lines.extend(_format_python_failure_details(results=python_node_results, style=style)) - if failure_lines: - lines.extend(failure_lines) - - skipped_lines: list[str] = _format_skipped_rollup( - result=result, python_node_results=python_node_results, style=style - ) - if skipped_lines: - lines.extend(skipped_lines) - - return "\n".join(lines) - - -def _format_skipped_rollup( - *, - result: BuildExecutionResult, - python_node_results: tuple[PythonNodeExecutionResult, ...], - style: CliStyle, -) -> list[str]: - """Render a dim rollup of nodes skipped because upstream work failed.""" - - skipped_names: list[str] = [ - model_result.model_name - for model_result in result.model_results - if model_result.status == ExecutionStatus.SKIPPED - ] - skipped_names.extend( - seed_result.seed_name - for seed_result in result.seed_results - if seed_result.status == ExecutionStatus.SKIPPED - ) - skipped_names.extend( - function_result.function_name - for function_result in result.function_results - if function_result.status == ExecutionStatus.SKIPPED - ) - skipped_names.extend( - load_result.source_name - for load_result in result.load_results - if load_result.status == ExecutionStatus.SKIPPED - ) - skipped_names.extend( - python_result.node_name - for python_result in python_node_results - if python_result.status == PythonNodeStatus.SKIPPED - ) - if not skipped_names: - return [] - visible_names: list[str] = skipped_names[:_SKIPPED_ROLLUP_CAP] - remaining_count: int = len(skipped_names) - len(visible_names) - rendered: str = ", ".join(visible_names) - if remaining_count > 0: - rendered = f"{rendered}, ... (+{remaining_count} more)" - return [ - "", - style.muted(f"Skipped ({len(skipped_names)}): {rendered}"), - ] - - -def _format_python_failure_details( - *, results: tuple[PythonNodeExecutionResult, ...], style: CliStyle -) -> list[str]: - lines: list[str] = [] - failed_results: tuple[PythonNodeExecutionResult, ...] = tuple( - result for result in results if result.status == PythonNodeStatus.FAILED - ) - if not failed_results: - return lines - lines.append("") - lines.append(style.error_strong("Python node failures:")) - lines.append("") - result: PythonNodeExecutionResult - for result in failed_results: - message: str = result.error_message or "Python node failed" - lines.append(f" {result.kind.value:<10}{result.node_name:<50} {style.error(message)}") - return lines - - -def _format_load_failure_details( - *, results: tuple[LoadExecutionResult, ...], style: CliStyle -) -> list[str]: - lines: list[str] = [] - load_result: LoadExecutionResult - for load_result in results: - if load_result.status != ExecutionStatus.FAILED: - continue - if not lines: - lines.append("") - lines.append(style.error_strong("Failures:")) - lines.append("") - lines.append(f" {load_result.source_name} ({load_result.resource_kind.value})") - if load_result.error_message is not None: - lines.extend( - _format_failure_error_block( - error_code=None, - error_message=load_result.error_message, - error_help=None, - style=style, - ) - ) - lines.append("") - return lines - - -def _format_failure_details(*, result: BuildExecutionResult, style: CliStyle) -> list[str]: - lines: list[str] = _format_load_failure_details(results=result.load_results, style=style) - has_failures: bool = bool(lines) - - seed_result: SeedExecutionResult - for seed_result in result.seed_results: - if seed_result.status != ExecutionStatus.FAILED: - continue - if not has_failures: - lines.append("") - lines.append(style.error_strong("Failures:")) - lines.append("") - has_failures = True - lines.append(f" {seed_result.seed_name} (seed)") - if seed_result.error_message is not None: - lines.extend( - _format_failure_error_block( - error_code=seed_result.error_code, - error_message=seed_result.error_message, - error_help=seed_result.error_help, - style=style, - ) - ) - lines.append("") - - model_result: ModelExecutionResult - for model_result in result.model_results: - if model_result.status != ExecutionStatus.FAILED: - continue - if not has_failures: - lines.append("") - lines.append(style.error_strong("Failures:")) - lines.append("") - has_failures = True - phase_str: str = f" ({model_result.failed_phase})" if model_result.failed_phase else "" - lines.append(f" {model_result.model_name}{phase_str}") - if model_result.error_message is not None: - lines.extend( - _format_failure_error_block( - error_code=model_result.error_code, - error_message=model_result.error_message, - error_help=model_result.error_help, - style=style, - ) - ) - if model_result.staging_relation is not None: - lines.append(f" {_inspection_relation_message(model_result.staging_relation)}") - lines.append("") - - function_result: FunctionExecutionResult - for function_result in result.function_results: - if function_result.status != ExecutionStatus.FAILED: - continue - if not has_failures: - lines.append("") - lines.append(style.error_strong("Failures:")) - lines.append("") - has_failures = True - lines.append(f" {function_result.function_name} ({function_result.function_kind})") - if function_result.error_message is not None: - lines.extend( - _format_failure_error_block( - error_code=function_result.error_code, - error_message=function_result.error_message, - error_help=function_result.error_help, - style=style, - ) - ) - lines.append("") - - test_r: SqlTestExecutionResult - for test_r in result.test_results: - if test_r.outcome == SqlTestOutcome.PASS: - continue - if not has_failures: - lines.append("") - lines.append(style.error_strong("Failures:")) - lines.append("") - has_failures = True - test_label: str = format_parameterized_test_label( - name=test_r.test_name, - source_path=test_r.source_path, - parameter_schema=test_r.parameter_schema, - parameter_values=test_r.parameter_values, - ) - lines.append(f" {test_label} (test)") - if test_r.error_message is not None: - lines.extend( - _format_failure_error_block( - error_code=test_r.error_code, - error_message=test_r.error_message, - error_help=test_r.error_help, - style=style, - ) - ) - lines.append("") - - lines.extend(_format_audit_error_details(result=result, style=style, has_failures=has_failures)) - return lines - - -def _format_audit_error_details( - *, result: BuildExecutionResult, style: CliStyle, has_failures: bool -) -> list[str]: - lines: list[str] = [] - audit_result: AuditExecutionResult - for audit_result in (*result.source_audit_results, *result.end_audit_results): - if audit_result.execution_error is None: - continue - if not has_failures: - lines.append("") - lines.append(style.error_strong("Failures:")) - lines.append("") - has_failures = True - target: str = ( - f" on {audit_result.attached_target_name}" - if audit_result.attached_target_name is not None - else "" - ) - lines.append(f" {audit_result.audit_name}{target} (audit)") - lines.extend( - _format_failure_error_block( - error_code=None, - error_message=audit_result.execution_error, - error_help=None, - style=style, - ) - ) - lines.append("") - - return lines - - -def _format_warning_details( - *, - result: BuildExecutionResult, - python_node_results: tuple[PythonNodeExecutionResult, ...], - style: CliStyle, -) -> list[str]: - lines: list[str] = [] - has_warnings: bool = False - - model_result: ModelExecutionResult - for model_result in result.model_results: - model_warnings: list[str] = [] - audit_r: AuditExecutionResult - for audit_r in model_result.audit_results: - if audit_r.outcome == AuditOutcome.WARN: - name: str = audit_r.audit_name - if audit_r.attached_column_name: - name = f"{audit_r.audit_name} ({audit_r.attached_column_name})" - if audit_r.evaluation_mode == AuditEvaluationMode.MEASUREMENT: - model_warnings.append(f" audit {name} measured {audit_r.measured_value}") - else: - row_label: str = "row" if audit_r.row_count == 1 else "rows" - model_warnings.append( - f" audit {name} returned {audit_r.row_count} {row_label}" - ) - warning_msg: str - for warning_msg in model_result.warning_messages: - model_warnings.append(f" {warning_msg}") - if model_warnings: - if not has_warnings: - lines.append("") - lines.append(style.warning_strong("Warnings:")) - lines.append("") - has_warnings = True - lines.append(f" {model_result.model_name}") - line: str - for line in model_warnings: - lines.append(line) - lines.append("") - - labelled_warnings: tuple[tuple[str, tuple[str, ...]], ...] = ( - *( - (f"{function_result.function_name} ({function_result.function_kind})", messages) - for function_result in result.function_results - if (messages := function_result.warning_messages) - ), - *( - (f"{load_result.source_name} (loader {load_result.loader_name})", messages) - for load_result in result.load_results - if (messages := load_result.warning_messages) - ), - *( - (f"{python_result.node_name} ({python_result.kind.value})", messages) - for python_result in python_node_results - if (messages := python_result.warning_messages) - ), - ) - label: str - messages: tuple[str, ...] - for label, messages in labelled_warnings: - if not has_warnings: - lines.append("") - lines.append(style.warning_strong("Warnings:")) - lines.append("") - has_warnings = True - lines.append(f" {label}") - for warning_msg in messages: - lines.append(f" {warning_msg}") - lines.append("") - - return lines - - -def _inspection_relation_message(relation_name: str) -> str: - if relation_name.endswith("__delta"): - return f"delta table kept for inspection: {relation_name}" - return f"staging table kept for inspection: {relation_name}" - - -def _format_failure_error_block( - *, error_code: str | None, error_message: str, error_help: str | None, style: CliStyle -) -> list[str]: - lines: list[str] = [] - label_padding: str = " " * max(0, _TYPE_WIDTH - len("error")) - label: str = f"{style.error_muted('error')}{label_padding}" - message: str = _format_result_error( - error_code=error_code, - error_message=error_message, - error_help=error_help, - use_color=style.use_color, - ) - formatted_line: str - for index, formatted_line in enumerate(message.splitlines() or [message]): - display_label: str = label if index == 0 else " " * _TYPE_WIDTH - lines.append(f" {display_label}{formatted_line}") - return lines - - -def _format_result_error( - *, error_code: str | None, error_message: str, error_help: str | None, use_color: bool -) -> str: - if error_code is None: - return error_message - return format_coded_error( - code=error_code, - message=error_message, - help=error_help, - use_color=use_color, - include_error_label=False, - ) - - -def _format_duration(duration_ms: int | None) -> str: - if duration_ms is None: - return "" - seconds: float = duration_ms / 1000.0 - seconds_per_minute: int = 60 - if seconds < seconds_per_minute: - return f"{seconds:.2f}s" - minutes: int = int(seconds // 60) - remaining: float = seconds - minutes * 60 - return f"{minutes}m{remaining:.1f}s" - - -def _execution_status_display(status: ExecutionStatus) -> str: - if status == ExecutionStatus.SUCCESS: - return "OK" - if status == ExecutionStatus.FAILED: - return "FAIL" - if status == ExecutionStatus.SKIPPED: - return "SKIP" - return str(status) - - -def _load_result_detail(result: LoadExecutionResult) -> str: - if result.status != ExecutionStatus.SKIPPED: - return f" rows={result.rows_loaded:,}" - skip_label: str = "skip" - if result.skip_mode is not None: - skip_label = f"{result.skip_mode.value} skip" - if result.skip_reason: - return f" {skip_label}: {result.skip_reason}" - return f" {skip_label}" - - -def _model_skip_detail(result: ModelExecutionResult) -> str: - skip_label: str = "skip" - if result.skip_mode is not None: - skip_label = f"{result.skip_mode.value} skip" - if result.skip_reason: - return f" {skip_label}: {result.skip_reason}" - return f" {skip_label}" - - -def _hook_skip_detail(result: HookExecutionResult) -> str: - if result.status != ExecutionStatus.SKIPPED: - return "" - skip_label: str = "skip" - if result.skip_mode is not None: - skip_label = f"{result.skip_mode.value} skip" - if result.skip_reason: - return f" {skip_label}: {result.skip_reason}" - return f" {skip_label}" - - -def _audit_outcome_display(outcome: AuditOutcome) -> str: - if outcome == AuditOutcome.PASS: - return "PASS" - if outcome == AuditOutcome.WARN: - return "WARN" - if outcome == AuditOutcome.ERROR: - return "FAIL" - if outcome == AuditOutcome.INSUFFICIENT: - return "INSUFFICIENT" - return str(outcome) - - -def _test_outcome_display(outcome: SqlTestOutcome) -> str: - if outcome == SqlTestOutcome.PASS: - return "PASS" - if outcome == SqlTestOutcome.ERROR: - return "ERROR" - return "FAIL" - - -def _aggregate_audit_results( - audit_results: tuple[AuditExecutionResult, ...], -) -> list[AuditDisplayEntry]: - """Aggregate per-batch delta audit results into single display entries.""" - - has_delta: bool = any(a.run_scope_phase == AuditRunScope.DELTA_AND_FINAL for a in audit_results) - - groups: dict[tuple[str, str | None, str], list[AuditExecutionResult]] = {} - audit: AuditExecutionResult - for audit in audit_results: - key: tuple[str, str | None, str] = ( - audit.audit_name, - audit.attached_column_name, - audit.run_scope_phase, - ) - groups.setdefault(key, []).append(audit) - - entries: list[AuditDisplayEntry] = [] - results: list[AuditExecutionResult] - for (_name, _col, _phase), results in groups.items(): - display_name: str = _name - if _col is not None: - display_name = f"{_name} ({_col})" - worst: AuditOutcome = _worst_audit_outcome(results) - total_rows: int = sum(r.row_count for r in results) - pass_count: int = sum(1 for r in results if r.outcome == AuditOutcome.PASS) - reused: bool = all(r.reused for r in results) - label: str = _phase_label( - phase=_phase, has_delta_audits=has_delta, batch_count=len(results) - ) - entries.append( - AuditDisplayEntry( - label=label, - display_name=display_name, - outcome=worst, - total_row_count=total_rows, - batch_pass=pass_count, - batch_total=len(results), - reused=reused, - executed_sql=results[0].executed_sql if results else None, - ) - ) - - return entries - - -def _worst_audit_outcome(results: list[AuditExecutionResult]) -> AuditOutcome: - """Return the worst outcome from a list of audit results.""" - - has_error: bool = any(r.outcome == AuditOutcome.ERROR for r in results) - if has_error: - return AuditOutcome.ERROR - has_warn: bool = any(r.outcome == AuditOutcome.WARN for r in results) - if has_warn: - return AuditOutcome.WARN - if any(r.outcome == AuditOutcome.INSUFFICIENT for r in results): - return AuditOutcome.INSUFFICIENT - return AuditOutcome.PASS - - -def _phase_label(*, phase: str, has_delta_audits: bool, batch_count: int) -> str: - """Return the audit type label, annotated with phase when delta audits are present.""" - - if not has_delta_audits: - return "audit" - if phase == AuditRunScope.DELTA_AND_FINAL: - return "audit (d)" - return "audit (f)" - - -def _resolve_verbose_events( - *, model_result: ModelExecutionResult, plan_entry: ModelPlanEntry | None -) -> tuple[LifeCycleEvent, ...]: - if model_result.lifecycle_events: - return model_result.lifecycle_events - if plan_entry is not None: - return (LifeCycleEvent(kind=LifeCycleEventKind.SQL, content=plan_entry.logical_ddl),) - return () - - -def _hook_label_width(hook_results: tuple[HookExecutionResult, ...]) -> int: - if not hook_results: - return _HOOK_MIN_LABEL_WIDTH - longest_label: int = max(len(result.label) for result in hook_results) - return max(_HOOK_MIN_LABEL_WIDTH, min(longest_label, _HOOK_MAX_LABEL_WIDTH)) - - -def _scheduler_diagnostic_message(state: SchedulerState) -> str: - if state.aborted: - return ( - "Scheduler " - f"{state.running} running, {state.ready} ready, " - f"{state.waiting} waiting on dependencies, {state.aborted} aborted after stop, " - f"limit {state.limit}" - ) - suffix: str = ( - " (DAG frontier constrained)" - if state.running < state.limit and state.ready == 0 and state.waiting > 0 - else "" - ) - return ( - "Scheduler " - f"{state.running} running, {state.ready} ready, " - f"{state.waiting} waiting on dependencies, limit {state.limit}{suffix}" - ) - - -def _query_diagnostic_message(telemetry: StatementExecutionTelemetry) -> str: - query_id: str = telemetry.query_id or "query ID unavailable" - return ( - "Query " - f"{telemetry.resource_type} {telemetry.resource_name} phase={telemetry.phase} " - f"{query_id} {telemetry.status} {telemetry.elapsed_seconds:.2f}s" - ) - - -def _format_display_sql(sql: str) -> str: - stripped: str = sql.rstrip() - if not stripped: - return sql - if stripped.endswith(";"): - return stripped - return f"{stripped};" - - -def _tree_child_name_length(name: str) -> int: - """Measure a tree child name so its status still fits the shared status column.""" - - return len(name) + _TREE_CHILD_NAME_INDENT - _NAME_PADDING - - -def _truncate_name(*, name: str, width: int) -> str: - """Truncate a display name to fit within the given width.""" - - if len(name) <= width: - return name - return name[: width - 3] + "..." - - -def _format_streaming_batch_summary(*, model_result: ModelExecutionResult) -> str | None: - """Return the batch summary line for streaming output, if applicable.""" - - if model_result.batch_count is None: - return None - batch_label: str = "batch" if model_result.batch_count == 1 else "batches" - count_text: str = f"{model_result.batch_count} {batch_label}" - if model_result.batch_size is not None: - count_text = f"{count_text} ({model_result.batch_size})" - parts: list[str] = [count_text] - if ( - model_result.cursor_range_start is not None - and model_result.cursor_range_end is not None - and not model_result.empty_cursor_inputs - ): - start: str = cursor_bound_display( - value=model_result.cursor_range_start, - cursor_type=model_result.cursor_type, - cursor_grain=model_result.cursor_grain, - ) - end: str = inclusive_cursor_end( - end=model_result.cursor_range_end, - cursor_type=model_result.cursor_type, - cursor_grain=model_result.cursor_grain, - ) - parts.append(f"range {start} \u2192 {end}") - if model_result.rows_affected is not None: - parts.append(_format_abbreviated_rows(count=model_result.rows_affected)) - if model_result.empty_cursor_inputs and not model_result.waiting_on_empty_inputs: - parts.append(format_no_input_rows(model_result.empty_cursor_inputs)) - if model_result.microbatch_recovery_batch_count: - parts.append(f"{model_result.microbatch_recovery_batch_count} recovery") - if model_result.microbatch_synthetic_completion_count: - parts.append(f"{model_result.microbatch_synthetic_completion_count} synthetic coverage") - if model_result.microbatch_replay_requirement_state is not None: - parts.append(f"replay {model_result.microbatch_replay_requirement_state}") - return f" {' '.join(parts)}" - - -def _format_empty_cursor_inputs(*, model_result: ModelExecutionResult) -> str | None: - """Return why a cursor window selected no rows, if its inputs were empty.""" - - if not model_result.empty_cursor_inputs or model_result.waiting_on_empty_inputs: - return None - return f" {format_no_input_rows(model_result.empty_cursor_inputs)}" - - -def _format_abbreviated_rows(*, count: int) -> str: - """Format a row count with K/M/B abbreviation above 10,000.""" - - label: str = "row" if count == 1 else "rows" - if count >= _ROW_COUNT_BILLION: - return f"{count / _ROW_COUNT_BILLION:.1f}B {label}" - if count >= _ROW_COUNT_MILLION: - return f"{count / _ROW_COUNT_MILLION:.1f}M {label}" - if count >= _ROW_COUNT_ABBREVIATION_THRESHOLD: - return f"{count / _ROW_COUNT_THOUSAND:.1f}K {label}" - return f"{count:,} {label}" diff --git a/src/sqlbuild/cli/commands/classes/dbt_init_progress_reporter.py b/src/sqlbuild/cli/commands/classes/dbt_init_progress_reporter.py index e8b39b8c31..ce30fe04a4 100644 --- a/src/sqlbuild/cli/commands/classes/dbt_init_progress_reporter.py +++ b/src/sqlbuild/cli/commands/classes/dbt_init_progress_reporter.py @@ -5,7 +5,7 @@ import time from typing import TextIO -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter +from sqlbuild._native import TransientStatusReporter class DbtInitProgressReporter: diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 30b98977e7..cab2bdd5c2 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -29,7 +29,6 @@ SkillUpdateResult, WrittenTarget, ) -from sqlbuild.cli.progress.models import AuditDisplayEntry, ExecutionCounts # noqa: F401 from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.cost.types import CostStatus from sqlbuild.spec.contracts.models import ( diff --git a/src/sqlbuild/cli/progress/classes/__init__.py b/src/sqlbuild/cli/progress/classes/__init__.py deleted file mode 100644 index 055ba47d35..0000000000 --- a/src/sqlbuild/cli/progress/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""CLI progress classes.""" diff --git a/src/sqlbuild/cli/progress/classes/audit_progress_reporter.py b/src/sqlbuild/cli/progress/classes/audit_progress_reporter.py deleted file mode 100644 index b6f2bc1551..0000000000 --- a/src/sqlbuild/cli/progress/classes/audit_progress_reporter.py +++ /dev/null @@ -1,198 +0,0 @@ -"""Coordinator-safe progress reporting for standalone concurrent audits.""" - -from __future__ import annotations - -import threading -from collections.abc import Callable -from typing import TextIO - -from sqlbuild._native import ( - NativeProgressProjector, - current_native_progress_projector, -) -from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.compiler.planner.models import AuditPlanEntry -from sqlbuild.executor.auditing.main.resource_id import audit_resource_id -from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator -from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator - -_LABEL_WIDTH: int = 10 -_NAME_WIDTH: int = 50 - - -class AuditProgressReporter: - """Render aggregate live state and flush completed audit rows in plan order.""" - - def __init__( - self, - *, - entries: tuple[AuditPlanEntry, ...], - worker_limit: int, - stream: TextIO, - use_color: bool, - ) -> None: - self._entries: tuple[AuditPlanEntry, ...] = entries - self._worker_limit: int = worker_limit - self._stream: TextIO = stream - self._lines: TransientLineCoordinator = shared_transient_line_coordinator() - self._style: CliStyle = CliStyle(use_color=use_color) - self._is_tty: bool = hasattr(stream, "isatty") and stream.isatty() - self._projector: NativeProgressProjector | None = current_native_progress_projector() - self._indexes: dict[str, int] = { - _entry_resource_id(entry): index for index, entry in enumerate(entries) - } - self._completed_by_index: dict[int, AuditExecutionResult | None] = {} - self._next_index: int = 0 - self._running: int = 0 - self._completed: int = 0 - self._current_group: str | None = None - self._lock: threading.RLock = self._lines.lock - self._closed: bool = False - self._cursor_hidden: bool = False - self._on_result: Callable[[AuditExecutionResult], None] | None = None - - def set_result_callback(self, callback: Callable[[AuditExecutionResult], None]) -> None: - """Install plan-ordered result enrichment before dispatch starts.""" - - self._on_result = callback - - def on_item_start(self, entry: AuditPlanEntry) -> None: - """Record a worker start without mutating one-item spinner state.""" - - with self._lock: - self._running += 1 - self._render_aggregate() - - def on_item_complete(self, result: AuditExecutionResult) -> None: - """Flush every newly contiguous plan-order result row.""" - - with self._lock: - self._completed_by_index[self._indexes[_result_resource_id(result)]] = result - self._flush_ready() - self._render_aggregate() - - def on_item_physical_complete(self, result: AuditExecutionResult) -> None: - """Update aggregate state immediately without projecting a result row.""" - - del result - with self._lock: - self._running = max(0, self._running - 1) - self._completed += 1 - self._render_aggregate() - - def on_item_error(self, entry: AuditPlanEntry) -> None: - """Advance aggregate and ordering state for a fatal audit exception.""" - - with self._lock: - self._running = max(0, self._running - 1) - self._completed += 1 - self._completed_by_index[self._indexes[_entry_resource_id(entry)]] = None - self._flush_ready() - self._render_aggregate() - - def close(self) -> None: - """Restore the terminal after the final aggregate update.""" - - with self._lock: - self._closed = True - self._lines.release(owner=self) - if self._is_tty: - self._clear_aggregate() - if self._cursor_hidden: - self._stream.write("\033[?25h") - self._cursor_hidden = False - self._stream.flush() - - def _flush_ready(self) -> None: - while self._next_index in self._completed_by_index: - result: AuditExecutionResult | None = self._completed_by_index.pop(self._next_index) - self._next_index += 1 - if result is not None: - self._write_result(result) - - def _write_result(self, result: AuditExecutionResult) -> None: - if self._is_tty: - self._clear_aggregate() - group_name: str = result.attached_target_name or "(unattached)" - if group_name != self._current_group: - prefix: str = "\n" if self._current_group is not None else "" - self._stream.write(f"{prefix}{self._style.section(group_name)}\n") - self._current_group = group_name - status_text: str = { - AuditOutcome.PASS: "PASS", - AuditOutcome.WARN: "WARN", - AuditOutcome.ERROR: "FAIL", - AuditOutcome.INSUFFICIENT: "INSUFFICIENT", - }[result.outcome] - name: str = result.audit_name - if result.attached_column_name is not None: - name = f"{name} ({result.attached_column_name})" - detail: str = "" - if result.outcome != AuditOutcome.PASS and result.row_count > 0: - row_label: str = "row" if result.row_count == 1 else "rows" - detail = f" {result.row_count} {row_label}" - if self._projector is not None: - duration_ms: float | None = self._projector.consume_resource_terminal( - resource_name=result.audit_name, - resource_id=_result_resource_id(result), - ) - if duration_ms is not None: - detail = f"{detail} {duration_ms / 1000.0:.2f}s" - status: str = self._style.status(status=status_text) - self._stream.write(f" {'audit':<{_LABEL_WIDTH}}{name:<{_NAME_WIDTH}} {status}{detail}\n") - self._stream.flush() - if self._on_result is not None: - self._on_result(result) - - def clear_transient_line(self) -> None: - """Erase the aggregate row so a persistent line can take its place.""" - - with self._lock: - self._clear_aggregate() - self._stream.flush() - - def redraw_transient_line(self) -> None: - """Draw the aggregate row again below persistent output.""" - - with self._lock: - self._render_aggregate() - - def _render_aggregate(self) -> None: - if not self._is_tty or self._closed: - return - self._lines.claim(stream=self._stream, owner=self) - if not self._cursor_hidden: - self._stream.write("\033[?25l") - self._cursor_hidden = True - self._clear_aggregate() - self._stream.write( - "\r\033[K audit " - f"running {self._running} | completed {self._completed}/{len(self._entries)} " - f"| workers {self._worker_limit}" - ) - self._stream.flush() - - def _clear_aggregate(self) -> None: - self._stream.write("\r\033[K") - - -def _entry_resource_id(entry: AuditPlanEntry) -> str: - return audit_resource_id( - audit_name=entry.name, - attachment_kind=entry.attachment_kind, - attached_target_kind=entry.attached_target_kind, - attached_target_name=entry.attached_target_name, - attached_column_name=entry.attached_column_name, - ) - - -def _result_resource_id(result: AuditExecutionResult) -> str: - return audit_resource_id( - audit_name=result.audit_name, - attachment_kind=result.attachment_kind, - attached_target_kind=result.attached_target_kind, - attached_target_name=result.attached_target_name, - attached_column_name=result.attached_column_name, - ) diff --git a/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py b/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py deleted file mode 100644 index 8b6c6ec396..0000000000 --- a/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py +++ /dev/null @@ -1,288 +0,0 @@ -"""Progress rendering for grouped audit and test style commands.""" - -from __future__ import annotations - -import threading -from typing import TextIO - -from sqlbuild._native import ( - NativeProgressProjector, - current_native_progress_projector, -) -from sqlbuild.cli.progress.constants import NESTED_CHILD_NAME_INDENT -from sqlbuild.cli.progress.models import NestedProgressChildRow -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator -from sqlbuild.presentation.main.inline_error_lines import format_inline_error_lines -from sqlbuild.presentation.main.terminal_columns import terminal_columns -from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator -from sqlbuild.presentation.main.tree_connector import tree_connector - -_LABEL_WIDTH: int = 10 -_NAME_WIDTH: int = 50 -_SPINNER_TICK_SECONDS: float = 0.1 -_LIVE_ROW_PADDING: int = 7 -_ACTIVE_SPINNER_FRAMES: tuple[str, ...] = ( - "⠋", - "⠙", - "⠹", - "⠸", - "⠼", - "⠴", - "⠦", - "⠧", - "⠇", - "⠏", -) - - -class NestedCommandProgressCallbacks: - """Render one live spinner row for all running items and grouped completed rows.""" - - def __init__( - self, - *, - total: int, - label: str, - stream: TextIO, - use_color: bool, - name_width: int = _NAME_WIDTH, - concurrency: int = 1, - ) -> None: - self._total: int = total - self._label: str = label - self._stream: TextIO = stream - self._lines: TransientLineCoordinator = shared_transient_line_coordinator() - self._use_color: bool = use_color - self._style: CliStyle = CliStyle(use_color=use_color) - self._name_width: int = max(_NAME_WIDTH, name_width) - self._is_tty: bool = hasattr(stream, "isatty") and stream.isatty() - self._sequential: bool = concurrency <= 1 - self._printed_group: str | None = None - self._running: dict[str, str] = {} - self._spinner_frame_index: int = 0 - self._write_lock: threading.RLock = self._lines.lock - self._spinner_line_active: bool = False - self._spinner_stop_event: threading.Event | None = None - self._spinner_thread: threading.Thread | None = None - self._cursor_hidden: bool = False - self._projector: NativeProgressProjector | None = current_native_progress_projector() - - def on_item_start( - self, - *, - group_name: str, - item_name: str, - canonical_resource_name: str | None = None, - ) -> None: - if self._projector is not None: - self._projector.expect_resource_enrichment( - resource_name=canonical_resource_name or item_name - ) - with self._write_lock: - if self._sequential and not self._running: - self._clear_live_row() - self._write_group_header(group_name) - self._running[item_name] = group_name - if self._is_tty: - self._hide_cursor() - self._spinner_line_active = True - self._lines.claim(stream=self._stream, owner=self) - self._draw_spinner_line() - self._ensure_spinner_loop() - - def on_item_complete( - self, - *, - group_name: str, - item_name: str, - status_text: str, - detail: str = "", - error_code: str | None = None, - error_help: str | None = None, - error_message: str | None = None, - child_rows: tuple[NestedProgressChildRow, ...] = (), - canonical_resource_name: str | None = None, - canonical_resource_id: str | None = None, - ) -> None: - duration_ms: float | None = None - if self._projector is not None: - duration_ms = self._projector.consume_resource_terminal( - resource_name=canonical_resource_name or item_name, - resource_id=canonical_resource_id, - ) - stopped_thread: threading.Thread | None - with self._write_lock: - item_group: str = self._running.pop(item_name, group_name) - if self._projector is None or duration_ms is not None: - if duration_ms is not None: - detail = f"{detail} {duration_ms / 1000.0:.2f}s" - self._clear_live_row() - self._write_group_header(item_group) - self._write_completed_item( - item_name=item_name, - status_text=status_text, - detail=detail, - error_code=error_code, - error_help=error_help, - error_message=error_message, - child_rows=child_rows, - ) - stopped_thread = self._refresh_live_row() - self._stream.flush() - if stopped_thread is not None and stopped_thread.is_alive(): - stopped_thread.join(timeout=0.2) - - def _write_group_header(self, group_name: str) -> None: - if group_name == self._printed_group: - return - prefix: str = "" if self._printed_group is None else "\n" - self._printed_group = group_name - self._stream.write(f"{prefix}{self._style.section(group_name)}\n") - self._stream.flush() - - def _write_completed_item( - self, - *, - item_name: str, - status_text: str, - detail: str, - error_code: str | None, - error_help: str | None, - error_message: str | None, - child_rows: tuple[NestedProgressChildRow, ...], - ) -> None: - status: str = self._style.status(status=status_text) - self._stream.write( - f" {self._label:<{_LABEL_WIDTH}}{item_name:<{self._name_width}} {status}{detail}\n" - ) - child_name_width: int = self._name_width - NESTED_CHILD_NAME_INDENT - child_row: NestedProgressChildRow - for index, child_row in enumerate(child_rows): - child_status: str = self._style.status(status=child_row.status_text) - last: bool = index == len(child_rows) - 1 and error_message is None - connector: str = tree_connector(style=self._style, last=last) - self._stream.write( - f" {connector} {child_row.label:<{_LABEL_WIDTH - 2}}" - f"{child_row.name:<{child_name_width}} " - f"{child_status}{child_row.detail}\n" - ) - if error_message is not None: - self._write_error( - error_code=error_code, - error_message=error_message, - error_help=error_help, - ) - - def _clear_live_row(self) -> None: - if self._is_tty and self._spinner_line_active: - self._stream.write("\r\033[K") - - def _refresh_live_row(self) -> threading.Thread | None: - """Redraw the live row for remaining items, or retire it; return a thread to join.""" - - if not self._is_tty: - return None - if self._running: - self._draw_spinner_line() - return None - self._spinner_line_active = False - self._lines.release(owner=self) - self._show_cursor() - stopped_thread: threading.Thread | None = self._spinner_thread - if self._spinner_stop_event is not None: - self._spinner_stop_event.set() - self._spinner_stop_event = None - self._spinner_thread = None - return stopped_thread - - def _write_error( - self, *, error_code: str | None, error_message: str, error_help: str | None - ) -> None: - pad: str = " " * 6 - connector: str = tree_connector(style=self._style, last=True) - label_width: int = _LABEL_WIDTH - 2 - label: str = self._style.error_muted(f"{'error':<{label_width}}") - plain_prefix_width: int = len(pad) + 4 + label_width - content_width: int = max(terminal_columns() - plain_prefix_width, 1) - lines: list[str] = format_inline_error_lines( - error_code=error_code, - error_message=error_message, - error_help=error_help, - content_width=content_width, - style=self._style, - ) - continuation_pad: str = " " * plain_prefix_width - for index, line in enumerate(lines): - if index == 0: - self._stream.write(f"{pad}{connector} {label}{line}\n") - else: - self._stream.write(f"{continuation_pad}{line}\n") - - def clear_transient_line(self) -> None: - """Erase the live spinner row so a persistent line can take its place.""" - - with self._write_lock: - self._stream.write("\r\033[K") - self._stream.flush() - - def redraw_transient_line(self) -> None: - """Draw the live spinner row again below persistent output.""" - - self._write_spinner_line() - - def _write_spinner_line(self) -> None: - with self._write_lock: - if self._spinner_line_active: - self._draw_spinner_line() - - def _draw_spinner_line(self) -> None: - spinner: str = self._style.status(status=_ACTIVE_SPINNER_FRAMES[self._spinner_frame_index]) - self._spinner_frame_index = (self._spinner_frame_index + 1) % len(_ACTIVE_SPINNER_FRAMES) - running: tuple[str, ...] = tuple(self._running) - text: str = ( - running[0] if len(running) == 1 else f"{len(running)} running: {', '.join(running)}" - ) - name_width: int = max( - min(self._name_width, terminal_columns() - _LABEL_WIDTH - _LIVE_ROW_PADDING), 1 - ) - name: str = text if len(text) <= name_width else f"{text[: max(name_width - 3, 0)]}..." - self._stream.write( - f"\r\033[K {self._label:<{_LABEL_WIDTH}}{name:<{name_width}} {spinner}" - ) - self._stream.flush() - - def _ensure_spinner_loop(self) -> None: - if self._spinner_thread is not None: - return - stop_event: threading.Event = threading.Event() - self._spinner_stop_event = stop_event - spinner_thread: threading.Thread = threading.Thread( - target=self._spin_until_stopped, - args=(stop_event,), - daemon=True, - ) - self._spinner_thread = spinner_thread - spinner_thread.start() - - def _spin_until_stopped(self, stop_event: threading.Event) -> None: - while not stop_event.wait(_SPINNER_TICK_SECONDS): - with self._write_lock: - if not stop_event.is_set() and self._spinner_line_active: - self._draw_spinner_line() - - def _hide_cursor(self) -> None: - if self._cursor_hidden: - return - with self._write_lock: - self._stream.write("\033[?25l") - self._stream.flush() - self._cursor_hidden = True - - def _show_cursor(self) -> None: - if not self._cursor_hidden: - return - with self._write_lock: - self._stream.write("\033[?25h") - self._stream.flush() - self._cursor_hidden = False diff --git a/src/sqlbuild/cli/progress/constants.py b/src/sqlbuild/cli/progress/constants.py deleted file mode 100644 index fb554beaa8..0000000000 --- a/src/sqlbuild/cli/progress/constants.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Shared layout constants for CLI progress rendering.""" - -NESTED_CHILD_NAME_INDENT: int = 4 diff --git a/src/sqlbuild/cli/progress/models.py b/src/sqlbuild/cli/progress/models.py index cbefa6e379..91c51b4c99 100644 --- a/src/sqlbuild/cli/progress/models.py +++ b/src/sqlbuild/cli/progress/models.py @@ -5,7 +5,6 @@ from dataclasses import dataclass from sqlbuild._native import ConnectionProgressReporter, PlanningProgressReporter -from sqlbuild.compiler.auditing.types import AuditOutcome @dataclass(frozen=True) @@ -24,31 +23,3 @@ class NestedProgressChildRow: name: str status_text: str detail: str = "" - - -@dataclass(frozen=True) -class AuditDisplayEntry: - """Aggregated audit result for display.""" - - label: str - display_name: str - outcome: AuditOutcome - total_row_count: int - batch_pass: int - batch_total: int - reused: bool = False - executed_sql: str | None = None - - -@dataclass(frozen=True) -class ExecutionCounts: - """Aggregated pass, warning, failure, and skip counts.""" - - pass_count: int = 0 - warn_count: int = 0 - fail_count: int = 0 - skip_count: int = 0 - - @property - def total_count(self) -> int: - return self.pass_count + self.warn_count + self.fail_count + self.skip_count diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/binding_requests.py b/src/sqlbuild/compiler/compile/_helpers/analysis/binding_requests.py deleted file mode 100644 index 35c2a57dbd..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/binding_requests.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Native schema-validation requests for model SQL.""" - -from __future__ import annotations - -from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql -from sqlbuild.compiler.sql_analysis.models import ( - SqlSchemaValidationRequest, -) - - -def get_complete_schema_binding_request( - *, - query_sql: str, - placeholders: dict[str, str] | None, - dialect: str | None, - binding_schema: dict[str, dict[str, str]], - known_functions: tuple[str, ...] = (), - known_types: tuple[str, ...] = (), - cleaned_sql: str | None = None, -) -> SqlSchemaValidationRequest: - """Build one stable native schema-validation request.""" - - if cleaned_sql is None: - cleaned_sql = normalize_analysis_sql( - sql=query_sql, dialect=dialect, placeholders=placeholders - ) - return SqlSchemaValidationRequest( - sql=cleaned_sql, - dialect=dialect, - schema=binding_schema, - known_functions=known_functions, - known_types=known_types, - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py b/src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py deleted file mode 100644 index 1b7373d28d..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Fixture-column inference implementation.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild._native import infer_native_query_columns -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns -from sqlbuild.compiler.compile.models import ( - FixtureColumnInference, - InferredColumn, -) - - -def infer_fixture_column_facts( - *, query_sql: str, inference_profile: ExpressionInferenceProfile -) -> FixtureColumnInference | None: - """Infer fixture columns and identify explicitly untyped NULL projections.""" - - analysis: NativeQueryColumns = infer_native_query_columns( - queries=(NativeColumnQuery(sql=query_sql, mode="legacy", recover_cte_facts=False),), - profile=inference_profile, - )[0] - if not analysis.succeeded or analysis.has_star or analysis.columns is None: - return None - simple_types, null_literal_names, quoted_names = _simple_projection_facts( - query_sql=query_sql, - dialect=inference_profile.sql_analysis_dialect, - ) - return FixtureColumnInference( - columns=tuple( - InferredColumn( - name=column.name, - type=column.type or simple_types.get(column.name.casefold()), - nullability=column.nullability, - ) - for column in analysis.columns - ), - null_literal_names=null_literal_names, - quoted_names=quoted_names, - ) - - -def _simple_projection_facts( - *, query_sql: str, dialect: str | None -) -> tuple[dict[str, str], frozenset[str], frozenset[str]]: - types: list[tuple[str, str]] - null_literal_names: list[str] - quoted_names: list[str] - types, null_literal_names, quoted_names = _native.fixture_projection_facts(query_sql, dialect) - return dict(types), frozenset(null_literal_names), frozenset(quoted_names) diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/reference_names.py b/src/sqlbuild/compiler/compile/_helpers/analysis/reference_names.py deleted file mode 100644 index 7f929085c3..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/reference_names.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Reference analysis names, lineage resource types, qualified-reference scans and placeholders.""" - -from __future__ import annotations - -import sqlbuild._native as _native - - -def table_function_analysis_name(function_name: str) -> str: - """Return the stable relation stub used to analyze a table-function call.""" - - return f"__sqlbuild_table_function_{function_name}" - - -def substitute_placeholder_defaults(*, query_sql: str, placeholders: dict[str, str]) -> str: - """Replace @@@name tokens with their default values for SQL analysis parsing.""" - - if not placeholders: - return query_sql - return _native.substitute_placeholder_defaults(query_sql, list(placeholders.items())) diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py b/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py deleted file mode 100644 index 6265087cef..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Help for SQL the parser rejects.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.constants import SQL_ANALYSIS_OPT_OUT_ENTRY -from sqlbuild.compiler.discovery.constants import ( - PROJECT_CONFIG_FILENAME, - SETTINGS_SECTION, - SQL_ANALYSIS_CONFIG_KEY, -) -from sqlbuild.errors.setting_help.main.join_helps import join_helps -from sqlbuild.errors.setting_help.main.model_header_help import model_header_help -from sqlbuild.errors.setting_help.main.setting_help import setting_help - - -def sql_analysis_opt_out_help(*, model_owned: bool = True) -> str: - """Help for SQL the parser rejects: the exact header, project and single-run opt-outs.""" - - report: str = ( - "if this SQL is valid for your warehouse, please report it so the parser can support it" - ) - return join_helps( - model_header_help( - purpose="if this SQL is valid for your warehouse, skip SQL analysis for this model", - entry=SQL_ANALYSIS_OPT_OUT_ENTRY, - follow_up="and please report the SQL so the parser can support it", - ) - if model_owned - else report, - setting_help( - purpose="SQL analysis is on for this project; to turn it off for every model", - file_name=PROJECT_CONFIG_FILENAME, - section=SETTINGS_SECTION, - key=SQL_ANALYSIS_CONFIG_KEY, - value=False, - ), - "to skip SQL analysis for one run, use `--no-sql-analysis`", - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/config/dynamic_columns.py b/src/sqlbuild/compiler/compile/_helpers/config/dynamic_columns.py deleted file mode 100644 index 9978f375c3..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/config/dynamic_columns.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Dynamic column-family configuration parsing for model headers.""" - -from __future__ import annotations diff --git a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py deleted file mode 100644 index 435a4ef422..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compile-time validation of model config combinations.""" - -from __future__ import annotations diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py deleted file mode 100644 index 907ed49485..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Complete semantic diagnostics: type recovery, metadata checks, recovery and explanations.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild._native import complete_semantic_project -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.types import DiagnosticRow -from sqlbuild.compiler.compile.types import CompiledProjectView - - -def complete_semantic_diagnostics( - *, - project: CompiledProjectView, - profile: ExpressionInferenceProfile, - binding_results: dict[str, list[DiagnosticRow]], - resource_sql_analysis: bool = True, - native_session: Any | None = None, -) -> CompiledProjectView: - """Recover type facts before metadata checks, then explain retained root diagnostics.""" - - return complete_semantic_project( - project=project, - profile=profile, - binding_results=binding_results, - resource_sql_analysis=resource_sql_analysis, - session=native_session, - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py deleted file mode 100644 index c04e76824c..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Hand audit, SQL test and SQL hook SQL to the native resource SQL checks.""" - -from __future__ import annotations - -from typing import Any, cast - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import CompiledProjectView - - -def get_resource_sql_diagnostics( - *, - project: CompiledProjectView, - shapes: dict[str, dict[str, str]], - profile: ExpressionInferenceProfile, -) -> tuple[CompilerDiagnostic, ...]: - """Bind audits, SQL tests and SQL hooks against the same closed shapes as models.""" - - return _native.resource_sql_diagnostics_of( - cast(Any, project.binding_catalog).native, project, shapes, profile - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py deleted file mode 100644 index 5a907dab0f..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Hand compiled-project assembly and model analysis to their native stages when enabled.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild._native import ( - analyze_native_model_sql, - infer_native_expression_source_shapes, - model_syntax_checks, - prove_native_dynamic_contracts, - standalone_pivot_models, -) -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.models import ( - NativeModelAnalyses, - NativeModelAnalysisRequest, - NativePivotTables, -) -from sqlbuild.compiler.analysis_session.types import ProofRow -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.project_assembly.main._assemble_native_project_resources import ( - assemble_native_project_resources, -) -from sqlbuild.compiler.project_assembly.models import ( - NativeModelFacts, - NativeProjectFacts, - NativeProjectResources, -) -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -def native_project_facts( - *, - inputs: CompileInputsView, - dialect: str | None, - analysis_model_names: frozenset[str] | None, - analyses: NativeModelAnalyses | None, - session: Any | None, - column_types_by_table: dict[str, dict[str, str]], - authoritative_column_types_by_table: dict[str, dict[str, str]], - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], -) -> NativeProjectFacts: - """Return the native project facts.""" - - resources: NativeProjectResources = assemble_native_project_resources( - inputs=inputs, - dialect=dialect, - syntax_checks=model_syntax_checks( - model_inputs=inputs.model_inputs, - analysis_model_names=analysis_model_names, - analysis_succeeded=frozenset() if analyses is None else analyses.succeeded, - ), - ) - pivots: dict[int, tuple[str, tuple[SchemaDynamicColumnFamily, ...]]] = standalone_pivot_models( - model_inputs=inputs.model_inputs, analysed={} if analyses is None else analyses.positions - ) - proofs: dict[int, ProofRow | None] = dict( - zip( - pivots, - prove_native_dynamic_contracts( - session=session, - tables=NativePivotTables( - dialect=dialect, - column_types_by_table=column_types_by_table, - authoritative_column_types_by_table=authoritative_column_types_by_table, - column_nullability_by_table=column_nullability_by_table, - dynamic_families_by_table=dynamic_families_by_table, - ), - models=tuple(pivots.values()), - ), - strict=True, - ) - ) - return NativeProjectFacts( - models=tuple( - NativeModelFacts( - deps=resources.model_deps[index], - dynamic_contract=proofs.get(index), - syntax_valid=resources.model_syntax_valid[index], - ) - for index in range(len(inputs.model_inputs)) - ), - resources=resources, - ) - - -def expression_source_shapes( - *, expressions: tuple[str, ...], profile: ExpressionInferenceProfile -) -> tuple[dict[str, str] | None, ...]: - """Infer one shape per expression source natively.""" - - return infer_native_expression_source_shapes(expressions=expressions, profile=profile) - - -def analyze_model_sql(request: NativeModelAnalysisRequest) -> NativeModelAnalyses: - """Analyze models natively; their outcomes stay in the finished session.""" - - return analyze_native_model_sql(request=request) diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py deleted file mode 100644 index 19364899f1..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Assemble every SQL test natively; macro-mocked tests' model queries run the user's macros.""" - -from __future__ import annotations - -import decimal -from collections.abc import Callable -from dataclasses import replace - -import sqlbuild._native as _native -from sqlbuild._native import assemble_native_sql_tests -from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompiledModelSqlTestPayload, - CompiledSqlTest, - CompileModelSqlTestInputPayload, - CompileSqlTestInput, -) -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.sql_test_glue.constants import ( - SQL_TEST_DECIMAL_OVERFLOW, - SQL_TEST_INPUT_FAILURE, -) -from sqlbuild.compiler.sql_test_glue.models import NativeSqlTestAssembly - - -def assemble_sql_tests( - *, - inputs: CompileInputsView, - macro_mock_queries: Callable[[CompileSqlTestInput], dict[str, str]], -) -> tuple[CompiledSqlTest, ...]: - """Assemble every SQL test natively, raising errors in Python's assembly order.""" - - native: tuple[NativeSqlTestAssembly, ...] = assemble_native_sql_tests(inputs=inputs) - tests: list[CompiledSqlTest] = [] - for test_input, assembly in zip(inputs.test_inputs, native, strict=True): - if assembly.test is None: - message: str = assembly.failure.message if assembly.failure is not None else "" - if assembly.failure is not None and assembly.failure.kind == SQL_TEST_INPUT_FAILURE: - raise CompileInputError(message) - raise _native.NativeCompilerError(message) - for key, diagnostic in assembly.diagnostics: - report_compile_diagnostic(key=key, diagnostic=diagnostic) - test: CompiledSqlTest = assembly.test - payload: object = test_input.payload - if isinstance(payload, CompileModelSqlTestInputPayload) and payload.macro_mocks: - test = _with_model_queries(test=test, queries=macro_mock_queries(test_input)) - if assembly.failure is not None and assembly.failure.kind == SQL_TEST_DECIMAL_OVERFLOW: - raise decimal.Overflow([decimal.Overflow]) - tests.append(test) - return tuple(tests) - - -def _with_model_queries(*, test: CompiledSqlTest, queries: dict[str, str]) -> CompiledSqlTest: - payload: object = test.payload - if not isinstance(payload, CompiledModelSqlTestPayload): - return test - return replace(test, payload=replace(payload, model_query_overrides=queries)) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py deleted file mode 100644 index 4f6baacade..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Compile-time context and template resolution for model header values.""" - -from __future__ import annotations - -from datetime import UTC, datetime -from typing import cast -from uuid import uuid4 - -from sqlbuild.compiler.compile._helpers.render.templating import expand_template_data -from sqlbuild.compiler.compile.types import CompileContextKey - - -def resolve_early_model_templates( - *, - values: dict[str, object], - effective_vars: dict[str, object], - effective_target_name: str | None, - run_id: str, -) -> dict[str, object]: - """Resolve `${name}`, `${ENV:...}`, and early `run.*` model templates.""" - - return cast( - dict[str, object], - expand_template_data( - value=values, - variables=effective_vars, - context_values={ - CompileContextKey.RUN_ID: run_id, - CompileContextKey.RUN_TARGET: effective_target_name, - }, - context_label="model config", - allow_context=True, - preserve_context_tokens=False, - preserve_unknown_context=True, - ), - ) - - -def resolve_run_id(*, selected_run_id: str | None) -> str: - """Resolve a stable compile invocation id.""" - - if selected_run_id is not None: - return selected_run_id - timestamp_prefix: str = datetime.now(tz=UTC).strftime("%Y%m%dT%H%M%SZ") - unique_suffix: str = uuid4().hex[:12] - return f"{timestamp_prefix}_{unique_suffix}" diff --git a/src/sqlbuild/compiler/compile/_helpers/render/cursor_intrinsics.py b/src/sqlbuild/compiler/compile/_helpers/render/cursor_intrinsics.py deleted file mode 100644 index f734a3b99a..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/render/cursor_intrinsics.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Recognition and rendering for model cursor-bound SQL intrinsics.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.planner.constants import ( - MICROBATCH_END_SENTINEL, - MICROBATCH_START_SENTINEL, -) -from sqlbuild.compiler.planner.types import CursorType - -_RESERVED_MARKERS: list[str] = [MICROBATCH_START_SENTINEL, MICROBATCH_END_SENTINEL] -_CURSOR_START_CALL: str = "__cursor_start()" -_CURSOR_END_CALL: str = "__cursor_end()" - - -def get_validated_model_cursor_intrinsics( - *, sql: str, config_values: dict[str, object], model_name: str -) -> str: - """Validate and canonicalize intrinsics in one model query.""" - - canonical_sql, error = _native.validated_model_cursor_intrinsics( - sql, - _RESERVED_MARKERS, - (model_name, config_values.get("materialized"), config_values.get("cursor")), - ) - if error is not None: - raise CompileInputError(error) - return str(canonical_sql) - - -def render_cursor_intrinsics(*, sql: str, start_sql: str, end_sql: str) -> str: - """Render recognized intrinsics to complete adapter-specific bound expressions.""" - - rendered, _ = _replaced(sql=sql, context="Model SQL", start_sql=start_sql, end_sql=end_sql) - return rendered - - -def cursor_intrinsics_analysis_sql(*, sql: str, cursor_type: object) -> str: - """Replace intrinsics with stable typed literals only for static SQL analysis.""" - - literal: str = ( - "CAST(0 AS BIGINT)" - if cursor_type == CursorType.INTEGER - else "CAST('2000-01-01 00:00:00' AS TIMESTAMP)" - ) - return render_cursor_intrinsics(sql=sql, start_sql=literal, end_sql=literal) - - -def has_cursor_intrinsics(sql: str) -> bool: - """Return whether executable SQL contains either cursor intrinsic.""" - - _, found = _replaced( - sql=sql, context="SQL", start_sql=_CURSOR_START_CALL, end_sql=_CURSOR_END_CALL - ) - return found - - -def _replaced(*, sql: str, context: str, start_sql: str, end_sql: str) -> tuple[str, bool]: - replaced, found, error = _native.replace_cursor_intrinsics(sql, context, (start_sql, end_sql)) - if error is not None: - raise CompileInputError(error) - return replaced, found - - -def cursor_intrinsic_semantics() -> list[str]: - """The reserved markers native cursor-intrinsic validation reads.""" - - return list(_RESERVED_MARKERS) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py deleted file mode 100644 index 8c1bfab4f3..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py +++ /dev/null @@ -1,222 +0,0 @@ -"""Compile-time resolution for enum and constant declarations.""" - -from __future__ import annotations - -from collections.abc import Mapping -from functools import partial -from pathlib import Path -from types import MappingProxyType - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - DeclarationExpansionResult, - DeclarationResolutionContext, - DeclarationRuntimeProjection, - DeclarationScopeResolver, - ExpansionSpan, - LoadedMacro, -) -from sqlbuild.compiler.compile.types import TypedSqlValueRenderer -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredConstantFile, - DiscoveredEnumFile, - DiscoveredProjectInputs, - DiscoveredSqlModelFile, - EnumDeclaration, -) -from sqlbuild.compiler.model_loop.main._build_native_declaration_contexts import ( - build_native_declaration_contexts, -) -from sqlbuild.compiler.model_loop.types import NativeDeclarationScan -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - ResourceIdentity, - ScopeIndex, - ScopeLookup, - UsageRecord, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - ResourceKind, - ScopeKind, -) -from sqlbuild.sql_values.exceptions import SqlValueRenderingError, SqlValueValidationError -from sqlbuild.sql_values.main.validate_rendered_size import validate_rendered_sql_value_size -from sqlbuild.sql_values.types import CollectionRendering, SqlValueKind - -_ACCEPTED_VALUES_AUDIT: str = "accepted_values" - - -def build_declaration_scope_resolver( - *, - discovered_inputs: DiscoveredProjectInputs, - scope_index: ScopeIndex, - loaded_macros: Mapping[str, LoadedMacro] | None = None, - lookup: ScopeLookup | None = None, -) -> DeclarationScopeResolver: - """Pair the serializable static index with original process-local declaration values.""" - - declarations: dict[ - DeclarationIdentity, EnumDeclaration | ConstantDeclaration | LoadedMacro - ] = {} - enum_file: DiscoveredEnumFile - for enum_file in discovered_inputs.enum_files: - for declaration in enum_file.declarations: - declarations[DeclarationIdentity(DeclarationKind.ENUM, declaration.name)] = declaration - constant_file: DiscoveredConstantFile - for constant_file in discovered_inputs.constant_files: - for declaration in constant_file.declarations: - declarations[DeclarationIdentity(DeclarationKind.CONSTANT, declaration.name)] = ( - declaration - ) - for model_file in discovered_inputs.model_files: - model_name_value: object = model_file.header_values.get("name") - model_name: str = ( - model_name_value - if isinstance(model_name_value, str) and model_name_value - else model_file.relative_path.stem - ) - owner: ResourceIdentity = ResourceIdentity(ResourceKind.MODEL, model_name) - for declaration in model_file.enum_declarations: - declarations[DeclarationIdentity(DeclarationKind.ENUM, declaration.name, owner)] = ( - declaration - ) - for declaration in model_file.constant_declarations: - declarations[DeclarationIdentity(DeclarationKind.CONSTANT, declaration.name, owner)] = ( - declaration - ) - if loaded_macros is not None: - for macro in loaded_macros.values(): - declarations[DeclarationIdentity(DeclarationKind.MACRO, macro.name)] = macro - scope_lookup: ScopeLookup = build_scope_lookup(index=scope_index) if lookup is None else lookup - return DeclarationScopeResolver( - project_dir=discovered_inputs.project_dir, - lookup=scope_lookup, - projection=DeclarationRuntimeProjection(declarations=MappingProxyType(declarations)), - resource_specific=frozenset( - declaration.identity.owner - for declaration in scope_index.declarations - if declaration.scope is ScopeKind.PRIVATE and declaration.identity.owner is not None - ) - | frozenset(grant.resource for grant in scope_index.grants), - native_contexts=build_native_declaration_contexts( - lookup=scope_lookup, declarations=declarations - ), - ) - - -def resolve_declaration_context( - *, - resolver: DeclarationScopeResolver, - file_path: Path, - resource: ResourceIdentity | None = None, -) -> DeclarationResolutionContext: - """Project canonical visibility for one authored path onto runtime declaration values.""" - - context: DeclarationResolutionContext - context, _ = resolver.native_contexts.resolve(resolver, file_path, resource) - return context - - -def build_model_declaration_indexes( - *, model_file: DiscoveredSqlModelFile -) -> tuple[dict[str, EnumDeclaration], dict[str, ConstantDeclaration]]: - """Build collision-checked declaration indexes private to one model.""" - - return ( - {declaration.name: declaration for declaration in model_file.enum_declarations}, - {declaration.name: declaration for declaration in model_file.constant_declarations}, - ) - - -def expand_scanned_declaration_references( # noqa: PLR0913 - *, - sql: str, - references: NativeDeclarationScan, - file_path: Path, - declarations: DeclarationResolutionContext, - value_renderer: TypedSqlValueRenderer, - collection_rendering: CollectionRendering, -) -> DeclarationExpansionResult: - """Splice natively scanned references and raise the first error the scan stopped at.""" - - scanned, stop = references - if not scanned and stop is None: - return DeclarationExpansionResult(sql=sql, spans=(), usages=()) - consumer: ResourceIdentity | DeclarationIdentity | None = declarations.consumer - expanded_sql, spans, usages = _native.expand_declaration_references( - (sql, list(scanned), stop), - declarations, - ( - consumer, - isinstance(consumer, ResourceIdentity) - and consumer.kind in {ResourceKind.TEST, ResourceKind.SCENARIO}, - str(file_path), - ), - ( - partial( - render_constant_declaration, - value_renderer=value_renderer, - collection_rendering=collection_rendering, - file_path=file_path, - ), - {"usage_record": UsageRecord, "compile_error": CompileInputError}, - ), - ) - return DeclarationExpansionResult( - sql=expanded_sql, - spans=tuple(ExpansionSpan(*span) for span in spans), - usages=tuple(dict.fromkeys(usages)), - ) - - -def render_constant_declaration( - *, - declaration: ConstantDeclaration, - value_renderer: TypedSqlValueRenderer, - collection_rendering: CollectionRendering, - file_path: Path | None = None, -) -> str: - """Render one validated constant with the active adapter's typed-value contract.""" - - selected_rendering: CollectionRendering = declaration.render_as or collection_rendering - try: - if declaration.value.kind in { - SqlValueKind.STRING, - SqlValueKind.INTEGER, - SqlValueKind.BOOLEAN, - SqlValueKind.FLOAT, - SqlValueKind.DECIMAL, - SqlValueKind.NULL, - }: - rendered: str = value_renderer.render_typed_scalar(value=declaration.value) - elif declaration.value.kind in {SqlValueKind.LIST, SqlValueKind.SET}: - rendered = ( - value_renderer.render_typed_array(value=declaration.value) - if selected_rendering == CollectionRendering.ARRAY - else value_renderer.render_typed_value_list(value=declaration.value) - ) - else: - rendered = value_renderer.render_typed_object(value=declaration.value) - validate_rendered_sql_value_size( - rendered_sql=rendered, - context=f"{declaration.relative_path} constant '{declaration.name}'", - ) - except (SqlValueRenderingError, SqlValueValidationError) as error: - raise CompileInputError( - f"{declaration.relative_path} constant '{declaration.name}' could not be rendered " - f"in '{file_path or declaration.relative_path}' by adapter " - f"'{value_renderer.adapter_name}' as " - f"{selected_rendering.value}: {error}", - ) from error - return rendered - - -def render_enum_member_value(*, value: str | int) -> str: - if isinstance(value, int): - return str(value) - escaped_value: str = value.replace("'", "''") - return f"'{escaped_value}'" diff --git a/src/sqlbuild/compiler/compile/_helpers/render/spans.py b/src/sqlbuild/compiler/compile/_helpers/render/spans.py deleted file mode 100644 index 4128bbdcc9..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/render/spans.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Map offsets in rendered SQL back onto the authored text that produced them.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import ExpansionSpan, MappedOffset - - -def map_output_offset(*, offset: int, spans: tuple[ExpansionSpan, ...]) -> MappedOffset: - """Resolve one rendered offset through a single expansion pass.""" - - mapped: int = offset - span: ExpansionSpan - for span in spans: - if offset < span.output_start: - break - if offset < span.output_end: - return MappedOffset(offset=span.source_start, generated=True) - mapped += (span.source_end - span.source_start) - (span.output_end - span.output_start) - return MappedOffset(offset=mapped, generated=False) - - -def map_through_passes( - *, offset: int, passes: tuple[tuple[ExpansionSpan, ...], ...] -) -> MappedOffset: - """Resolve one rendered offset back through every expansion pass in order.""" - - current: int = offset - generated: bool = False - spans: tuple[ExpansionSpan, ...] - for spans in reversed(passes): - resolved: MappedOffset = map_output_offset(offset=current, spans=spans) - current = resolved.offset - generated = generated or resolved.generated - return MappedOffset(offset=current, generated=generated) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py deleted file mode 100644 index 2e3cd8e2e6..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ /dev/null @@ -1,139 +0,0 @@ -"""SQL project variable and environment interpolation helpers.""" - -from __future__ import annotations - -from collections.abc import Mapping -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild._native import expand_sql_macros, render_project_var_text -from sqlbuild.compiler.compile._helpers.render.declarations import ( - expand_scanned_declaration_references, -) -from sqlbuild.compiler.compile._helpers.render.templating import record_template_reads -from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - DeclarationExpansionResult, - DeclarationResolutionContext, - DeclarationScopeResolver, - ExpansionSpan, - LoadedMacro, - MacroContext, - MacroExpansionResult, - SqlInterpolation, -) -from sqlbuild.compiler.compile.types import TypedSqlValueRenderer -from sqlbuild.compiler.model_loop.main._scan_native_declaration_references import ( - scan_native_declaration_references, -) -from sqlbuild.sql_values.types import CollectionRendering - -type _InterpolationRow = tuple[ - str | None, list[tuple[int, int, int, int]], list[tuple[str, str]], str | None -] - - -def expand_authored_sql_with_spans( - *, - sql: str, - file_path: Path, - effective_vars: dict[str, object], - loaded_macros: dict[str, LoadedMacro], - macro_context: MacroContext, - value_renderer: TypedSqlValueRenderer, - collection_rendering: CollectionRendering, - context_values: Mapping[str, str | None] | None = None, - declarations: DeclarationResolutionContext | None = None, - declaration_resolver: DeclarationScopeResolver | None = None, -) -> tuple[str, tuple[tuple[ExpansionSpan, ...], ...]]: - """Expand authored SQL, returning each pass's substitution spans in order.""" - - interpolated_sql: str - interpolation_spans: tuple[ExpansionSpan, ...] - interpolated_sql, interpolation_spans = substitute_sql_vars_with_spans( - sql=sql, - file_path=file_path, - effective_vars=effective_vars, - context_values=context_values, - ) - declaration_result: DeclarationExpansionResult = expand_scanned_declaration_references( - sql=interpolated_sql, - references=scan_native_declaration_references(sqls=(interpolated_sql,))[0], - file_path=file_path, - declarations=DeclarationResolutionContext( - enums=declarations.enums if declarations is not None else {}, - constants=declarations.constants if declarations is not None else {}, - inaccessible_enums=declarations.inaccessible_enums if declarations is not None else {}, - inaccessible_constants=( - declarations.inaccessible_constants if declarations is not None else {} - ), - ), - value_renderer=value_renderer, - collection_rendering=collection_rendering, - ) - declaration_expanded_sql: str = declaration_result.sql - declaration_spans: tuple[ExpansionSpan, ...] = declaration_result.spans - expansion: MacroExpansionResult = expand_sql_macros( - sql=declaration_expanded_sql, - file_path=file_path, - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_resolver=declaration_resolver, - ) - return expansion.sql, (interpolation_spans, declaration_spans, expansion.spans) - - -def substitute_sql_vars_with_spans( - *, - sql: str, - file_path: Path, - effective_vars: dict[str, object], - context_values: Mapping[str, str | None] | None = None, -) -> tuple[str, tuple[ExpansionSpan, ...]]: - """Replace SQL interpolation tokens, returning the span of every substitution.""" - - interpolation: SqlInterpolation = interpolate_sql_batch( - sqls=((sql, file_path),), effective_vars=effective_vars, context_values=context_values - )[0] - return applied_interpolation(interpolation), interpolation.spans - - -def interpolate_sql_batch( - *, - sqls: tuple[tuple[str, Path], ...], - effective_vars: dict[str, object], - context_values: Mapping[str, str | None] | None = None, -) -> tuple[SqlInterpolation, ...]: - """Interpolate every `(sql, file path)` natively; errors wait in their result.""" - - rows: list[_InterpolationRow] = _native.interpolate_sql_batch( - [(sql, str(file_path)) for sql, file_path in sqls], - (effective_vars, UnicodeEnvironment(), context_values, render_project_var_text), - ) - return tuple( - _interpolation(sql=sql, row=row) for (sql, _file_path), row in zip(sqls, rows, strict=True) - ) - - -def applied_interpolation(interpolation: SqlInterpolation) -> str: - """Record the interpolation's environment and context reads, then return or raise it.""" - - record_template_reads(interpolation.reads) - if interpolation.error is not None: - raise CompileInputError(interpolation.error) - return interpolation.sql - - -def _interpolation( - *, - sql: str, - row: _InterpolationRow, -) -> SqlInterpolation: - rendered, spans, reads, error = row - return SqlInterpolation( - sql=rendered if rendered is not None else sql, - spans=tuple(ExpansionSpan(*span) for span in spans), - reads=tuple(reads), - error=error, - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/templating.py b/src/sqlbuild/compiler/compile/_helpers/render/templating.py deleted file mode 100644 index ccdfcd649b..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/render/templating.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Template expansion for compile inputs, natively, with the reads it makes recorded.""" - -from __future__ import annotations - -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.model_config.constants import ENVIRONMENT_READ -from sqlbuild.compiler.model_config.main._expand_native_config_templates import ( - expand_native_config_templates, -) -from sqlbuild.compiler.model_config.main._native_config_error import native_config_error -from sqlbuild.compiler.model_config.models import ( - NativeTemplateExpansion, - NativeTemplateRejection, - TemplateResolutionFlags, -) - - -def expand_effective_vars(raw_values: dict[str, object]) -> dict[str, object]: - """Resolve merged effective vars with recursive `${name}` expansion.""" - - site: tuple[str, str, int] | None = _native.lone_surrogate_site(raw_values) - if site is not None: - name, path, byte_offset = site - raise CompileInputError( - f"Variable '{name}' holds a lone surrogate in {path} at UTF-8 byte " - f"{byte_offset}, which is not valid Unicode text", - help=( - f"Set '{name}' to valid Unicode text in --vars or the [vars] table; the " - "value is not shown because it may be a secret" - ), - ) - outcome: tuple[object, list[tuple[str, str]]] = _native.expand_effective_vars( - raw_values, UnicodeEnvironment() - ) - record_template_reads(tuple(outcome[1])) - if isinstance(outcome[0], _native.NativeConfigError): - raise native_config_error(error=outcome[0]) - return cast(dict[str, object], outcome[0]) - - -def expand_template_data( - *, - value: object, - variables: dict[str, object], - context_values: dict[str, str | None], - context_label: str, - allow_context: bool, - preserve_context_tokens: bool, - preserve_unknown_context: bool, -) -> object: - """Recursively expand template strings inside supported Python container values.""" - - outcome: NativeTemplateExpansion | NativeTemplateRejection = expand_native_config_templates( - value=value, - variables=variables, - context_values=context_values, - flags=TemplateResolutionFlags( - allow_context=allow_context, - preserve_context_tokens=preserve_context_tokens, - preserve_unknown_context=preserve_unknown_context, - ), - context_label=context_label, - ) - record_template_reads(outcome.reads) - if isinstance(outcome, NativeTemplateRejection): - raise native_config_error(error=outcome.error) - return outcome.value - - -def record_template_reads(reads: tuple[tuple[str, str], ...]) -> None: - """Record native template environment and context reads for project reuse, in order.""" - - for kind, name in reads: - if kind == ENVIRONMENT_READ: - COMPILE_INPUT_READS.environment_read(name) - else: - COMPILE_INPUT_READS.context_read(name) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py index e48e6316ca..7473eecc9d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py @@ -16,6 +16,7 @@ from typing import cast import sqlbuild._native as _native +from sqlbuild._native import validate_resource_identity from sqlbuild.compiler.compile.exceptions import ( CompileInputError, MacroDeclarationLookupError, @@ -33,9 +34,6 @@ DiscoveredMacroFile, EnumDeclaration, ) -from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( - validate_resource_identity, -) from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, @@ -373,7 +371,7 @@ def _rewrite_absolute_project_imports( return ast.Module(body=body, type_ignores=tree.type_ignores) -def _inventory_project_macros( +def inventory_project_macros( *, macro_files: tuple[DiscoveredMacroFile, ...] ) -> StaticMacroInventory: """Validate and inventory project macro exports without executing authored code.""" diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py deleted file mode 100644 index 47743fb45b..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py +++ /dev/null @@ -1,26 +0,0 @@ -"""SQL-native test compile-semantic extraction helpers.""" - -from __future__ import annotations - -from functools import lru_cache - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompileSqlTestCte, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -@lru_cache(maxsize=4096) -def extract_unclassified_sql_test_ctes( - *, sql: str, file_label: str, syntax: SqlLexicalSyntax -) -> tuple[CompileSqlTestCte, ...]: - """Extract raw top-level CTEs before mode-specific classification.""" - - error: str | None - ctes: list[tuple[str, str]] - error, ctes = _native.scope_test_ctes([(sql, file_label)], syntax.native_mapping)[0] - if error is not None: - raise CompileInputError(error) - return tuple(CompileSqlTestCte(name=name, sql_body=body) for name, body in ctes) diff --git a/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py b/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py index 8211248490..a01db8e479 100644 --- a/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py +++ b/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py @@ -2,9 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.compile._helpers.sql_tests.core import ( - extract_unclassified_sql_test_ctes, -) +from sqlbuild._native import extract_sql_test_ctes from sqlbuild.compiler.compile.models import CompileSqlTestCte from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @@ -18,4 +16,4 @@ def extract( ) -> tuple[CompileSqlTestCte, ...]: """Extract top-level SQL-test CTEs without classifying their roles.""" - return extract_unclassified_sql_test_ctes(sql=sql, file_label=file_label, syntax=syntax) + return extract_sql_test_ctes(sql=sql, file_label=file_label, syntax=syntax) diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 9e29853df1..dfc9b660ad 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -50,7 +50,6 @@ "command_output_sinks": "runtime command outputs, not graph resources", "native_session": "native parse state kept for later stages, not a project input", } -SQL_ANALYSIS_OPT_OUT_ENTRY: str = "sql_analysis false" HOOK_DIRECTORY_NAME: str = "hooks" MODEL_DIRECTORY_NAME: str = "models" NOT_NULL_AUDIT_NAME: str = "not_null" diff --git a/src/sqlbuild/compiler/compile/main/__init__.py b/src/sqlbuild/compiler/compile/main/__init__.py deleted file mode 100644 index 279e9ab38a..0000000000 --- a/src/sqlbuild/compiler/compile/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compile public entry modules.""" diff --git a/src/sqlbuild/compiler/compile/main/_analyze_columns_and_lineage.py b/src/sqlbuild/compiler/compile/main/_analyze_columns_and_lineage.py deleted file mode 100644 index bfae6b5bae..0000000000 --- a/src/sqlbuild/compiler/compile/main/_analyze_columns_and_lineage.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Public compile-time resolved column-read analysis operation.""" - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import ( - CompiledLineageSourceFact, - CompileSqlReference, -) - - -def analyze_resolved_column_reads( - *, - query_sql: str, - references: tuple[CompileSqlReference, ...], - dialect: str | None, -) -> tuple[CompiledLineageSourceFact, ...]: - """Return CTE-aware column reads resolved to external resources.""" - - reference_rows: list[tuple[str, str]] = [ - (str(reference.ref_kind), reference.ref_name) for reference in references - ] - reads: list[tuple[str, str, str]] = _native.resolve_required_columns( - query_sql, reference_rows, dialect - ) - return tuple( - CompiledLineageSourceFact( - resource_type=resource_type, resource_name=resource_name, column_name=column_name - ) - for resource_type, resource_name, column_name in reads - ) diff --git a/src/sqlbuild/compiler/compile/main/_effective_target.py b/src/sqlbuild/compiler/compile/main/_effective_target.py deleted file mode 100644 index 7d07777ee6..0000000000 --- a/src/sqlbuild/compiler/compile/main/_effective_target.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Effective target config helpers for CLI entrypoints.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.spec.contracts.models import TargetConfig - - -def build_effective_target_config( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None = None, -) -> TargetConfig | None: - """Build effective target config without compiling resources.""" - - target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=selected_target, - ) - if target_name is None: - return None - return resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ) diff --git a/src/sqlbuild/compiler/compile/main/_effective_target_namespace.py b/src/sqlbuild/compiler/compile/main/_effective_target_namespace.py deleted file mode 100644 index 87184dda4a..0000000000 --- a/src/sqlbuild/compiler/compile/main/_effective_target_namespace.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Public effective target namespace operation.""" - -from sqlbuild._native import ( - build_effective_target_namespace as _build_effective_target_namespace, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import TargetConfig - - -def build_effective_target_namespace( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None, - connection_database: object | None, - connection_schema: object | None, - default_database: str | None, - default_schema: str | None, -) -> tuple[str | None, TargetConfig | None, str | None, str | None]: - """Resolve the target and its effective database/schema without compiling resources.""" - - return _build_effective_target_namespace( - discovered_inputs=discovered_inputs, - selected_target=selected_target, - connection_database=connection_database, - connection_schema=connection_schema, - default_database=default_database, - default_schema=default_schema, - ) diff --git a/src/sqlbuild/compiler/compile/main/_infer_fixture_columns.py b/src/sqlbuild/compiler/compile/main/_infer_fixture_columns.py deleted file mode 100644 index b07f9e25de..0000000000 --- a/src/sqlbuild/compiler/compile/main/_infer_fixture_columns.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public fixture-column inference entrypoint.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile._helpers.analysis.fixture_columns import ( - infer_fixture_column_facts as _infer_fixture_column_facts, -) -from sqlbuild.compiler.compile.models import FixtureColumnInference - - -def infer_fixture_column_facts( - *, query_sql: str, inference_profile: ExpressionInferenceProfile -) -> FixtureColumnInference | None: - """Infer fixture columns and identify explicitly untyped NULL projections.""" - - return _infer_fixture_column_facts( - query_sql=query_sql, - inference_profile=inference_profile, - ) diff --git a/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py b/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py deleted file mode 100644 index 8ccdcc776a..0000000000 --- a/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Shared compile-input projection into a usage-enriched scope index.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.scopes.main._get_placement_validated_scope_index import ( - get_placement_validated_scope_index, -) -from sqlbuild.compiler.scopes.models import ScopeIndex, UsageRecord - - -def scope_index_with_compile_usages(*, inputs: CompileInputsView) -> ScopeIndex: - """Return compile inputs' canonical index with all usage and placement facts.""" - - collected_usages: list[UsageRecord] = list( - _native.model_schema_scope_usages( - inputs.discovered_inputs.model_schema_files, - inputs.declaration_scope.resolver if inputs.declaration_scope is not None else None, - ) - ) - for model_input in inputs.model_inputs: - collected_usages.extend(model_input.macro_usages) - collected_usages.extend(model_input.declaration_usages) - for source_input in inputs.source_inputs: - collected_usages.extend(source_input.declaration_usages) - for function_input in inputs.sql_function_inputs: - collected_usages.extend(function_input.declaration_usages) - for audit_input in inputs.audit_inputs: - collected_usages.extend(audit_input.declaration_usages) - for test_input in inputs.test_inputs: - collected_usages.extend(test_input.declaration_usages) - for scenario_input in inputs.scenario_inputs: - collected_usages.extend(scenario_input.declaration_usages) - return get_placement_validated_scope_index( - index=inputs.scope_index, - enforce_placement=inputs.project_config.scopes.enforce_placement, - compile_usages=tuple(collected_usages), - ) diff --git a/src/sqlbuild/compiler/compile/main/_semantic_coverage.py b/src/sqlbuild/compiler/compile/main/_semantic_coverage.py deleted file mode 100644 index e960e20a53..0000000000 --- a/src/sqlbuild/compiler/compile/main/_semantic_coverage.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Public semantic-check coverage report.""" - -from sqlbuild._native import semantic_coverage -from sqlbuild.compiler.compile.types import CompiledProjectView - - -def get_semantic_coverage(project: CompiledProjectView) -> dict[str, tuple[str, ...]]: - """Return per-model partial-check reasons.""" - return semantic_coverage(project) diff --git a/src/sqlbuild/compiler/compile/main/_source_bindings.py b/src/sqlbuild/compiler/compile/main/_source_bindings.py deleted file mode 100644 index 083f3e1c5a..0000000000 --- a/src/sqlbuild/compiler/compile/main/_source_bindings.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Planner entrypoint for warehouse-backed semantic binding.""" - -from sqlbuild._native import source_binding_diagnostics -from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompilerDiagnostic -from sqlbuild.compiler.compile.types import CompiledProjectView - - -def get_source_binding_diagnostics( - *, - project: CompiledProjectView, - columns: dict[str, tuple[ColumnInfo, ...]], - profile: ExpressionInferenceProfile, - selected_keys: frozenset[CompiledObjectKey], -) -> tuple[CompilerDiagnostic, ...]: - """Bind selected source readers to already-inspected warehouse columns.""" - return source_binding_diagnostics( - project=project, columns=columns, profile=profile, selected_keys=selected_keys - ) diff --git a/src/sqlbuild/compiler/compile/main/_static_macro_inventory.py b/src/sqlbuild/compiler/compile/main/_static_macro_inventory.py deleted file mode 100644 index b261bc641c..0000000000 --- a/src/sqlbuild/compiler/compile/main/_static_macro_inventory.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Compiler static macro inventory without authored module execution.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile._helpers.render.user_macros import _inventory_project_macros -from sqlbuild.compiler.compile.models import StaticMacroInventory -from sqlbuild.compiler.discovery.models import DiscoveredMacroFile - - -def inventory_project_macros( - *, macro_files: tuple[DiscoveredMacroFile, ...] -) -> StaticMacroInventory: - """Return validated AST exports and faults without executing macro modules.""" - - return _inventory_project_macros(macro_files=macro_files) diff --git a/src/sqlbuild/compiler/compile/main/cursor_intrinsics.py b/src/sqlbuild/compiler/compile/main/cursor_intrinsics.py deleted file mode 100644 index 2587bdf5ee..0000000000 --- a/src/sqlbuild/compiler/compile/main/cursor_intrinsics.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Published cursor-intrinsic compiler operation.""" - -from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( - has_cursor_intrinsics, - render_cursor_intrinsics, -) - - -def resolve_cursor_intrinsics( - *, sql: str, start_sql: str | None = None, end_sql: str | None = None -) -> tuple[str, bool]: - """Report intrinsic use and optionally render both bounds.""" - - found: bool = has_cursor_intrinsics(sql) - if not found or start_sql is None or end_sql is None: - return sql, found - return ( - render_cursor_intrinsics(sql=sql, start_sql=start_sql, end_sql=end_sql), - True, - ) diff --git a/src/sqlbuild/compiler/compile/main/effective_config.py b/src/sqlbuild/compiler/compile/main/effective_config.py deleted file mode 100644 index 5bb306fe43..0000000000 --- a/src/sqlbuild/compiler/compile/main/effective_config.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Effective project config helpers for compile and CLI entrypoints.""" - -from __future__ import annotations - -from sqlbuild._native import ( - build_effective_connection, - build_effective_vars, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.spec.contracts.models import TargetConfig - - -def build_effective_connection_config( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None = None, - cli_vars: dict[str, object] | None = None, -) -> dict[str, object]: - """Build the effective project connection config without compiling resources.""" - - target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=selected_target, - ) - target_config: TargetConfig | None = None - if target_name is not None: - target_config = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ) - effective_vars: dict[str, object] = build_effective_vars( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_config=target_config, - cli_vars={} if cli_vars is None else cli_vars, - ) - return build_effective_connection( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_config=target_config, - effective_vars=effective_vars, - ) diff --git a/src/sqlbuild/compiler/compile/main/effective_runtime.py b/src/sqlbuild/compiler/compile/main/effective_runtime.py deleted file mode 100644 index 719c6713cc..0000000000 --- a/src/sqlbuild/compiler/compile/main/effective_runtime.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Effective runtime config helpers for CLI entrypoints.""" - -from __future__ import annotations - -from sqlbuild._native import ( - build_effective_vars, -) -from sqlbuild.compiler.compile._helpers.render.context_templates import resolve_run_id -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.spec.contracts.models import TargetConfig - - -def build_effective_runtime_config( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None = None, - cli_vars: dict[str, object] | None = None, -) -> tuple[str | None, dict[str, object], str]: - """Build environment, vars, and invocation id without compiling resources.""" - - target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=selected_target, - ) - target_config: TargetConfig | None = None - if target_name is not None: - target_config = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ) - effective_vars: dict[str, object] = build_effective_vars( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_config=target_config, - cli_vars={} if cli_vars is None else cli_vars, - ) - return target_name, effective_vars, resolve_run_id(selected_run_id=None) diff --git a/src/sqlbuild/compiler/compile/main/effective_settings.py b/src/sqlbuild/compiler/compile/main/effective_settings.py deleted file mode 100644 index dbe8e63a9b..0000000000 --- a/src/sqlbuild/compiler/compile/main/effective_settings.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Effective project settings helper for compile and CLI entrypoints.""" - -from __future__ import annotations - -from sqlbuild._native import ( - build_effective_settings, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import SettingsConfig - - -def build_effective_settings_config( - *, discovered_inputs: DiscoveredProjectInputs -) -> SettingsConfig: - """Build effective settings without compiling resources.""" - - return build_effective_settings( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) diff --git a/src/sqlbuild/compiler/compile/main/expand_sql_with_spans.py b/src/sqlbuild/compiler/compile/main/expand_sql_with_spans.py deleted file mode 100644 index ec9f8da08e..0000000000 --- a/src/sqlbuild/compiler/compile/main/expand_sql_with_spans.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Expand one authored SQL body while reporting each substitution's span.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_context -from sqlbuild.compiler.compile._helpers.render.sql_vars import expand_authored_sql_with_spans -from sqlbuild.compiler.compile.models import ( - DeclarationResolutionContext, - ExpansionSpan, - SqlExpansionContext, -) - - -def expand_sql_with_spans( - *, sql: str, file_path: Path, context: SqlExpansionContext -) -> tuple[str, tuple[tuple[ExpansionSpan, ...], ...]]: - """Expand authored SQL and return each expansion pass's substitution spans.""" - - local_declarations: DeclarationResolutionContext | None = context.local_declarations.get( - file_path - ) - declarations: DeclarationResolutionContext = DeclarationResolutionContext( - enums=context.enums, - constants=context.constants, - ) - if context.declaration_resolver is not None: - declarations = resolve_declaration_context( - resolver=context.declaration_resolver, file_path=file_path - ) - if local_declarations is not None: - declarations = DeclarationResolutionContext( - enums=declarations.enums | local_declarations.enums, - constants=declarations.constants | local_declarations.constants, - inaccessible_enums=declarations.inaccessible_enums, - inaccessible_constants=declarations.inaccessible_constants, - ) - return expand_authored_sql_with_spans( - sql=sql, - file_path=file_path, - effective_vars=context.effective_vars, - loaded_macros=context.loaded_macros, - macro_context=context.macro_context, - declarations=declarations, - declaration_resolver=context.declaration_resolver, - value_renderer=context.value_renderer, - collection_rendering=context.collection_rendering, - ) diff --git a/src/sqlbuild/compiler/compile/main/expand_template_data.py b/src/sqlbuild/compiler/compile/main/expand_template_data.py deleted file mode 100644 index cd173d3d2f..0000000000 --- a/src/sqlbuild/compiler/compile/main/expand_template_data.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Public entrypoint for recursive template expansion.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile._helpers.render.templating import ( - expand_template_data as _expand_template_data, -) - - -def expand_template_data( - *, - value: object, - variables: dict[str, object], - context_values: dict[str, str | None], - context_label: str, - allow_context: bool, - preserve_context_tokens: bool, - preserve_unknown_context: bool, -) -> object: - """Recursively expand template strings inside supported Python container values.""" - - return _expand_template_data( - value=value, - variables=variables, - context_values=context_values, - context_label=context_label, - allow_context=allow_context, - preserve_context_tokens=preserve_context_tokens, - preserve_unknown_context=preserve_unknown_context, - ) diff --git a/src/sqlbuild/compiler/compile/main/map_expanded_offset.py b/src/sqlbuild/compiler/compile/main/map_expanded_offset.py deleted file mode 100644 index 4482858e88..0000000000 --- a/src/sqlbuild/compiler/compile/main/map_expanded_offset.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Resolve an offset in expanded SQL back onto the authored body.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile._helpers.render.spans import map_through_passes -from sqlbuild.compiler.compile.models import ExpansionSpan, MappedOffset - - -def map_expanded_offset( - *, offset: int, passes: tuple[tuple[ExpansionSpan, ...], ...] -) -> MappedOffset: - """Resolve an offset in expanded SQL back onto the authored body.""" - - return map_through_passes(offset=offset, passes=passes) diff --git a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py deleted file mode 100644 index 1978725d64..0000000000 --- a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Build the context required to expand a project's authored SQL bodies.""" - -from __future__ import annotations - -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild._native import ( - build_declaration_scope, - build_effective_vars, - discover_project_inputs, - rebind_declaration_scope, -) -from sqlbuild.compiler.compile._helpers.render.declarations import ( - build_model_declaration_indexes, -) -from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros -from sqlbuild.compiler.compile.models import ( - ConstantDeclaration, - DeclarationResolutionContext, - DeclarationScopeBuild, - EnumDeclaration, - LoadedMacro, - MacroContext, - SqlExpansionContext, -) -from sqlbuild.compiler.compile.types import TypedSqlValueRenderer -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( - resolve_effective_collection_rendering, -) - - -def build_sql_expansion_context( - *, - project_dir: Path, - value_renderer: TypedSqlValueRenderer, - cli_vars: dict[str, object] | None = None, - discovered_inputs: DiscoveredProjectInputs | None = None, - declaration_scope: DeclarationScopeBuild | None = None, - static_declaration_scope: DeclarationScopeBuild | None = None, -) -> SqlExpansionContext: - """Assemble SQL expansion inputs; a static scope lends only its index to private macros.""" - - effective_discovered_inputs: DiscoveredProjectInputs = ( - discovered_inputs - if discovered_inputs is not None - else discover_project_inputs(project_dir=project_dir, sql_analysis_enabled_override=False) - ) - effective_vars: dict[str, object] = build_effective_vars( - project_config=effective_discovered_inputs.project_config, - local_config=effective_discovered_inputs.local_config, - target_config=None, - cli_vars={} if cli_vars is None else cli_vars, - ) - scope: DeclarationScopeBuild = declaration_scope or _build_declaration_scope( - discovered_inputs=effective_discovered_inputs, - static_scope=static_declaration_scope, - sql_lexical_syntax=value_renderer.sql_lexical_syntax, - ) - loaded_macros: dict[str, LoadedMacro] = scope.loaded_macros - enums: dict[str, EnumDeclaration] - constants: dict[str, ConstantDeclaration] - enums, constants = _native.public_declaration_indexes(effective_discovered_inputs) - local_declarations: dict[Path, DeclarationResolutionContext] = {} - for model_file in effective_discovered_inputs.model_files: - local_enums: dict[str, EnumDeclaration] - local_constants: dict[str, ConstantDeclaration] - local_enums, local_constants = build_model_declaration_indexes(model_file=model_file) - local_declarations[model_file.file_path] = DeclarationResolutionContext( - enums=local_enums, - constants=local_constants, - ) - return SqlExpansionContext( - effective_vars=effective_vars, - loaded_macros=loaded_macros, - macro_context=MacroContext( - adapter_name=resolve_effective_adapter_name( - project_config=effective_discovered_inputs.project_config, - local_config=effective_discovered_inputs.local_config, - ), - sql_analysis_enabled=False, - target_name=effective_discovered_inputs.project_config.default_target, - vars=effective_vars, - _value_renderer=value_renderer, - _collection_rendering=resolve_effective_collection_rendering( - project_config=effective_discovered_inputs.project_config, - declaration_override=None, - ), - _enforce_explicit_references=( - effective_discovered_inputs.project_config.references.enforce_explicit - ), - ), - enums=enums, - constants=constants, - value_renderer=value_renderer, - collection_rendering=resolve_effective_collection_rendering( - project_config=effective_discovered_inputs.project_config, - declaration_override=None, - ), - local_declarations=local_declarations, - declaration_resolver=scope.resolver, - ) - - -def _build_declaration_scope( - *, - discovered_inputs: DiscoveredProjectInputs, - static_scope: DeclarationScopeBuild | None, - sql_lexical_syntax: SqlLexicalSyntax, -) -> DeclarationScopeBuild: - loaded_macros: dict[str, LoadedMacro] = load_project_macros(discovered_inputs.macro_files) - rebound: DeclarationScopeBuild | None = ( - None - if static_scope is None - else rebind_declaration_scope( - scope=static_scope, discovered_inputs=discovered_inputs, loaded_macros=loaded_macros - ) - ) - return rebound or build_declaration_scope( - discovered_inputs=discovered_inputs, - loaded_macros=loaded_macros, - sql_lexical_syntax=sql_lexical_syntax, - ) diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 59439b520e..d219e66130 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -343,9 +343,7 @@ def render_constant(self, name: str) -> str: raise CompileInputError( "Constant SQL rendering requires a declaration resolved for this macro invocation" ) - from sqlbuild.compiler.compile._helpers.render.declarations import ( - render_constant_declaration, - ) + from sqlbuild._native import render_constant_declaration return render_constant_declaration( declaration=declaration, @@ -357,9 +355,7 @@ def render_enum_member(self, *, enum_name: str, member_name: str) -> str: """Render one visible enum member as a SQL scalar literal.""" value: str | int = self.enums[enum_name][member_name] - from sqlbuild.compiler.compile._helpers.render.declarations import ( - render_enum_member_value, - ) + from sqlbuild._native import render_enum_member_value return render_enum_member_value(value=value) diff --git a/src/sqlbuild/compiler/contract_adoption/main/_write.py b/src/sqlbuild/compiler/contract_adoption/main/_write.py index fd0219ff50..7050ced037 100644 --- a/src/sqlbuild/compiler/contract_adoption/main/_write.py +++ b/src/sqlbuild/compiler/contract_adoption/main/_write.py @@ -11,6 +11,7 @@ edit_contract_model, edit_contract_source, remaining_contract_findings, + write_atomically, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( @@ -28,7 +29,6 @@ ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.lint.main.write_atomically import write_atomically def write_contracts( diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index 588848f6b6..c981bd9dc3 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Any, get_type_hints import sqlbuild._native as _native +from sqlbuild._native import validate_resource_identity from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import ( project_relative_path, @@ -42,9 +43,6 @@ DiscoveredPythonNodeFunctions, DiscoveredTaskFunction, ) -from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( - validate_resource_identity, -) from sqlbuild.provider.exceptions import ProviderInputError from sqlbuild.python_nodes.main.read_asset_definition import read_asset_definition from sqlbuild.python_nodes.main.read_audit_factory_definition import ( diff --git a/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py b/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py deleted file mode 100644 index 2f032de89e..0000000000 --- a/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Build a dbt-compatible manifest.json from a compiled project and its native graph.""" - -from __future__ import annotations - -from datetime import UTC, datetime - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.python_nodes.main._hook_identities import build_hook_identities - - -def build_graph_manifest( - *, - project: CompiledProjectView, - plan_output: PlanOutput | None, - project_name: str, - adapter_type: str, - graph: _native.NativeProjectGraph, -) -> _native.NativeManifest: - """Build the manifest with parent and child maps read from the native lineage graph.""" - - return _native.build_manifest( - project=project, - plan_output=plan_output, - graph=graph, - facts={ - "project_name": project_name, - "adapter_type": adapter_type, - "generated_at": datetime.now(tz=UTC).isoformat(), - "hook_identities": build_hook_identities(project.hook_functions), - }, - ) diff --git a/src/sqlbuild/compiler/manifest/main/build.py b/src/sqlbuild/compiler/manifest/main/build.py deleted file mode 100644 index 63ab9d03c3..0000000000 --- a/src/sqlbuild/compiler/manifest/main/build.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Build a dbt-compatible manifest.json from compiled project state.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild._native import native_graph_from_edges -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.manifest.main._build_graph_manifest import build_graph_manifest -from sqlbuild.compiler.planner.models import PlanOutput - - -def build_manifest( - *, - project: CompiledProjectView, - plan_output: PlanOutput | None = None, - project_name: str, - adapter_type: str, - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> _native.NativeManifest: - """Build a full dbt v12-compatible manifest from caller-built edge maps.""" - - graph: _native.NativeProjectGraph = native_graph_from_edges( - upstream=upstream_deps, downstream=downstream_deps - ) - return build_graph_manifest( - project=project, - plan_output=plan_output, - project_name=project_name, - adapter_type=adapter_type, - graph=graph, - ) diff --git a/src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py b/src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py deleted file mode 100644 index c955e8dd0a..0000000000 --- a/src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Expand `${...}` templates natively, returning the value or Python's error with its reads.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment -from sqlbuild.compiler.model_config.models import ( - NativeTemplateExpansion, - NativeTemplateRejection, - TemplateResolutionFlags, -) - - -def expand_native_config_templates( - *, - value: object, - variables: dict[str, object], - context_values: dict[str, str | None], - flags: TemplateResolutionFlags, - context_label: str, -) -> NativeTemplateExpansion | NativeTemplateRejection: - """Return the expanded value, or the error expansion stopped at; both carry their reads.""" - - outcome: tuple[object, list[tuple[str, str]]] = _native.expand_config_templates( - value, - (variables, UnicodeEnvironment(), context_values), - ( - flags.allow_context, - flags.preserve_context_tokens, - flags.preserve_unknown_context, - context_label, - ), - ) - if isinstance(outcome[0], _native.NativeConfigError): - return NativeTemplateRejection(error=outcome[0], reads=tuple(outcome[1])) - return NativeTemplateExpansion(value=outcome[0], reads=tuple(outcome[1])) diff --git a/src/sqlbuild/compiler/model_config/models.py b/src/sqlbuild/compiler/model_config/models.py deleted file mode 100644 index 1a95bb0fbd..0000000000 --- a/src/sqlbuild/compiler/model_config/models.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Natively parsed MODEL header metadata and template expansions.""" - -from __future__ import annotations - -from dataclasses import dataclass - -import sqlbuild._native as _native - - -@dataclass(frozen=True, slots=True) -class NativeTemplateExpansion: - """A natively expanded config value and the environment and context names it read, in order.""" - - value: object - reads: tuple[tuple[str, str], ...] - - -@dataclass(frozen=True, slots=True) -class NativeTemplateRejection: - """Python's exact template error for a config value and the names read before it, in order.""" - - error: _native.NativeConfigError - reads: tuple[tuple[str, str], ...] - - -@dataclass(frozen=True, slots=True) -class TemplateResolutionFlags: - """How one template expansion treats `CTX:` references.""" - - allow_context: bool - preserve_context_tokens: bool - preserve_unknown_context: bool diff --git a/src/sqlbuild/compiler/model_loop/__init__.py b/src/sqlbuild/compiler/model_loop/__init__.py deleted file mode 100644 index 91f5295966..0000000000 --- a/src/sqlbuild/compiler/model_loop/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""The native model compile loop.""" diff --git a/src/sqlbuild/compiler/model_loop/constants.py b/src/sqlbuild/compiler/model_loop/constants.py deleted file mode 100644 index f1e2ec1996..0000000000 --- a/src/sqlbuild/compiler/model_loop/constants.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Codes the native model loop shares with its Rust bindings.""" - -from sqlbuild.compiler.scopes.types import DeclarationKind - -DECLARATION_KIND_CODES: dict[DeclarationKind, int] = { - DeclarationKind.ENUM: 0, - DeclarationKind.CONSTANT: 1, - DeclarationKind.MACRO: 2, -} -NO_DECLARATION_KIND: int = 255 -ENUM_REFERENCE_KIND_CODE: int = DECLARATION_KIND_CODES[DeclarationKind.ENUM] -UNCLOSED_QUOTE_STOP_CODE: int = 0 -UNCLOSED_BLOCK_COMMENT_STOP_CODE: int = 1 -INVALID_ENUM_REFERENCE_STOP_CODE: int = 2 -INVALID_CONSTANT_REFERENCE_STOP_CODE: int = 3 diff --git a/src/sqlbuild/compiler/model_loop/main/__init__.py b/src/sqlbuild/compiler/model_loop/main/__init__.py deleted file mode 100644 index 1956248370..0000000000 --- a/src/sqlbuild/compiler/model_loop/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native model loop entry points.""" diff --git a/src/sqlbuild/compiler/model_loop/main/_build_native_declaration_contexts.py b/src/sqlbuild/compiler/model_loop/main/_build_native_declaration_contexts.py deleted file mode 100644 index b77a6a61eb..0000000000 --- a/src/sqlbuild/compiler/model_loop/main/_build_native_declaration_contexts.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Hand one scope lookup to the native declaration context classifier.""" - -from __future__ import annotations - -from collections.abc import Mapping - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import ( - DeclarationResolutionContext, - LoadedMacro, -) -from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration -from sqlbuild.compiler.model_loop.constants import DECLARATION_KIND_CODES, NO_DECLARATION_KIND -from sqlbuild.compiler.scopes.exceptions import ( - InvalidQualifiedIdentityError, - InvalidScopePathError, -) -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationVisibilityIndex, - OwnershipRoot, - ResourceIdentity, - ResourceRecord, - ScopeLookup, - VisibilityRecord, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - ResourceKind, - ScopeKind, - VisibilityReason, -) - - -def build_native_declaration_contexts( - *, - lookup: ScopeLookup, - declarations: Mapping[DeclarationIdentity, EnumDeclaration | ConstantDeclaration | LoadedMacro], -) -> _native.NativeDeclarationContexts: - """Hand the lookup's declaration positions and runtime values to the native classifier.""" - - index: DeclarationVisibilityIndex = lookup.visibility_index - identity_keys: dict[DeclarationIdentity, int] = {} - positions: list[tuple[object, ...]] = [] - for identity in index.identities: - key: int = identity_keys.setdefault(identity, len(identity_keys)) - value: EnumDeclaration | ConstantDeclaration | LoadedMacro | None = declarations.get( - identity - ) - positions.append( - ( - identity, - lookup.declarations[identity][0], - identity.name, - DECLARATION_KIND_CODES.get(identity.kind, NO_DECLARATION_KIND), - value, - _value_kind(value), - key, - ) - ) - return _native.NativeDeclarationContexts( - positions, - ( - list(index.global_positions), - {owner: list(items) for owner, items in index.local_positions.items()}, - {owner: list(items) for owner, items in index.inherited_positions.items()}, - ), - { - "visibility_record": VisibilityRecord, - "resolution_context": DeclarationResolutionContext, - "allocate": object.__new__, - "reasons": {reason.value: reason for reason in VisibilityReason}, - }, - ( - { - "identity_keys": identity_keys, - "private_positions": index.private_positions, - "grants_by_resource": lookup.grants_by_resource, - }, - { - "resources_by_path": lookup.resources_by_path, - "resources": lookup.resources, - "declarations": lookup.declarations, - "index_declarations": lookup.index.declarations, - "resource_record": ResourceRecord, - "ownership_root": OwnershipRoot, - "resource_identity": ResourceIdentity, - "declaration_identity": DeclarationIdentity, - "resource_kind": ResourceKind, - "declaration_kind": DeclarationKind, - "private_scope": ScopeKind.PRIVATE, - "identity_error": InvalidQualifiedIdentityError, - "path_error": InvalidScopePathError, - }, - ), - ) - - -def _value_kind(value: EnumDeclaration | ConstantDeclaration | LoadedMacro | None) -> int: - if isinstance(value, EnumDeclaration): - return DECLARATION_KIND_CODES[DeclarationKind.ENUM] - if isinstance(value, ConstantDeclaration): - return DECLARATION_KIND_CODES[DeclarationKind.CONSTANT] - if isinstance(value, LoadedMacro): - return DECLARATION_KIND_CODES[DeclarationKind.MACRO] - return NO_DECLARATION_KIND diff --git a/src/sqlbuild/compiler/model_loop/main/_scan_native_declaration_references.py b/src/sqlbuild/compiler/model_loop/main/_scan_native_declaration_references.py deleted file mode 100644 index 89ca8425a5..0000000000 --- a/src/sqlbuild/compiler/model_loop/main/_scan_native_declaration_references.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Scan `@enum`/`@const` references of many SQL strings in one native call.""" - -from __future__ import annotations - -from collections.abc import Callable - -import sqlbuild._native as _native -from sqlbuild.compiler.model_loop.types import NativeDeclarationScan - - -def scan_native_declaration_references( - *, sqls: tuple[str, ...], skip_malformed: bool = False -) -> tuple[NativeDeclarationScan, ...]: - """Return each string's references and stop code; `skip_malformed` walks past malformed ones.""" - - scan_references: Callable[ - [list[str]], - list[tuple[list[tuple[int, str, str | None, int, int]], int | None]], - ] = ( - _native.scan_sql_declaration_usages - if skip_malformed - else _native.scan_sql_declaration_references - ) - return tuple((tuple(scan[0]), scan[1]) for scan in scan_references(list(sqls))) diff --git a/src/sqlbuild/compiler/model_loop/types.py b/src/sqlbuild/compiler/model_loop/types.py deleted file mode 100644 index 703716e74c..0000000000 --- a/src/sqlbuild/compiler/model_loop/types.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Row shapes the native model loop exchanges with its Rust bindings.""" - -type NativeDeclarationReference = tuple[int, str, str | None, int, int] -type NativeDeclarationScan = tuple[tuple[NativeDeclarationReference, ...], int | None] diff --git a/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py b/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py deleted file mode 100644 index 596615324a..0000000000 --- a/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Selection policies applied to compile pipeline options.""" - -from __future__ import annotations - -from dataclasses import replace - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import ( - CompileAnalysisSelection, - CompiledObjectKey, -) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineOptions -from sqlbuild.compiler.planner.main.selection._resolve_planner_scopes import ( - resolve_planner_scopes, -) -from sqlbuild.compiler.planner.main.selection._split_unit_test_selectors import ( - split_project_unit_test_selectors, -) -from sqlbuild.compiler.planner.models import ( - PlannerPolicies, - PlannerScopeResolution, - PlannerSelection, - UnitTestSelectorSplit, -) - - -def apply_unit_test_selectors( - *, options: CompilePipelineOptions | None, discovered_inputs: DiscoveredProjectInputs -) -> CompilePipelineOptions: - """Return options whose selectors name only resources, with unit tests selected apart.""" - - resolved: CompilePipelineOptions = options if options is not None else CompilePipelineOptions() - if not resolved.select and not resolved.exclude: - return resolved - split: UnitTestSelectorSplit = split_project_unit_test_selectors( - select=resolved.select, - exclude=resolved.exclude, - discovered_inputs=discovered_inputs, - accepts_unit_tests=resolved.accepts_unit_test_selectors, - ) - return replace( - resolved, - select=split.select, - exclude=split.exclude, - sql_test_selection=split.sql_test_selection, - ) - - -def resolve_configured_rule_selection( - *, - options: CompilePipelineOptions, - discovered_inputs: DiscoveredProjectInputs, - project: CompiledProjectView, -) -> frozenset[CompiledObjectKey] | None: - """Align configured-rule subjects with models that received deep SQL analysis.""" - - if not options.sql_test_selection.models_selected: - return frozenset() - analysis_selection: CompileAnalysisSelection | None = ( - _native.resolve_compile_analysis_selection( - options=options, - discovered_inputs=discovered_inputs, - ) - ) - if analysis_selection is None or ( - not analysis_selection.select and not analysis_selection.exclude - ): - return None - scopes: PlannerScopeResolution = resolve_planner_scopes( - project=project, - selection=PlannerSelection( - select=analysis_selection.select, - exclude=analysis_selection.exclude, - ), - policies=PlannerPolicies(auto_load_sources=analysis_selection.auto_load_sources), - ) - return frozenset(model.key for model in scopes.stale_warning_scope.models_by_name.values()) - - -def resolve_analysis_model_names( - *, - compile_inputs: CompileInputsView, - inference_profile: ExpressionInferenceProfile, - selection: CompileAnalysisSelection | None, -) -> frozenset[str] | None: - """Return the models selected for deep analysis, or None to analyse every model.""" - - if selection is None or (not selection.select and not selection.exclude): - return None - structural_project: CompiledProjectView = _native.assemble_project( - inputs=compile_inputs, - inference_profile=inference_profile, - skip_column_inference=True, - analysis_model_names=frozenset(), - ) - scopes: PlannerScopeResolution = resolve_planner_scopes( - project=structural_project, - selection=PlannerSelection(select=selection.select, exclude=selection.exclude), - policies=PlannerPolicies(auto_load_sources=selection.auto_load_sources), - ) - return frozenset(scopes.stale_warning_scope.models_by_name) diff --git a/src/sqlbuild/compiler/pipeline/_helpers/clone.py b/src/sqlbuild/compiler/pipeline/_helpers/clone.py deleted file mode 100644 index aa31ac89d8..0000000000 --- a/src/sqlbuild/compiler/pipeline/_helpers/clone.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Clone pipeline assembly helpers.""" - -from __future__ import annotations - -from sqlbuild._native import build_compiled_project -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompileAnalysisSelection -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ( - ClonePipelineConnection, - ClonePipelineOptions, - ClonePipelineResult, -) -from sqlbuild.compiler.planner.main.clone._clone import run_clone_planning -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.models import TargetConfig - - -def prepare_clone_pipeline( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - origin_target_name: str, - destination_target_name: str, - destination_connection: ClonePipelineConnection, - options: ClonePipelineOptions, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, -) -> ClonePipelineResult: - origin_project: CompiledProjectView = _compile_project_for_environment( - discovered_inputs=discovered_inputs, - adapter=adapter, - target_name=origin_target_name, - resolved_connection=destination_connection.config, - no_sql_validation=options.no_sql_validation, - no_cache=options.no_cache, - cli_vars=options.cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - destination_project: CompiledProjectView = _compile_project_for_environment( - discovered_inputs=discovered_inputs, - adapter=adapter, - target_name=destination_target_name, - resolved_connection=destination_connection.config, - no_sql_validation=options.no_sql_validation, - no_cache=options.no_cache, - cli_vars=options.cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - origin_read_target_name: str = _read_target_name( - discovered_inputs=discovered_inputs, - target_name=origin_target_name, - ) - destination_read_target_name: str = _read_target_name( - discovered_inputs=discovered_inputs, - target_name=destination_target_name, - ) - origin_source_project: CompiledProjectView = ( - origin_project - if origin_read_target_name == origin_target_name - else _compile_project_for_environment( - discovered_inputs=discovered_inputs, - adapter=adapter, - target_name=origin_read_target_name, - resolved_connection=destination_connection.config, - no_sql_validation=options.no_sql_validation, - no_cache=options.no_cache, - cli_vars=options.cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - ) - destination_source_project: CompiledProjectView = ( - destination_project - if destination_read_target_name == destination_target_name - else _compile_project_for_environment( - discovered_inputs=discovered_inputs, - adapter=adapter, - target_name=destination_read_target_name, - resolved_connection=destination_connection.config, - no_sql_validation=options.no_sql_validation, - no_cache=options.no_cache, - cli_vars=options.cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - ) - ( - clone_plan, - destination_source_entries, - destination_model_entries, - destination_seed_entries, - destination_function_entries, - origin_model_entries, - origin_seed_entries, - origin_source_entries, - ) = run_clone_planning( - project=destination_project, - select=options.select, - exclude=options.exclude, - adapter=adapter, - connection=destination_connection.handle, - origin_project=origin_project, - origin_source_project=origin_source_project, - destination_source_project=destination_source_project, - ) - return ClonePipelineResult( - origin_project=origin_project, - destination_project=destination_project, - clone_plan=clone_plan, - destination_source_entries=destination_source_entries, - destination_model_entries=destination_model_entries, - destination_seed_entries=destination_seed_entries, - destination_function_entries=destination_function_entries, - origin_model_entries=origin_model_entries, - origin_seed_entries=origin_seed_entries, - origin_source_entries=origin_source_entries, - ) - - -def _compile_project_for_environment( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - target_name: str, - resolved_connection: dict[str, object], - no_sql_validation: bool, - no_cache: bool, - cli_vars: dict[str, object] | None, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None, -) -> CompiledProjectView: - return build_compiled_project( - discovered_inputs=discovered_inputs, - adapter=adapter, - selected_target=target_name, - resolved_connection=resolved_connection, - no_sql_validation=no_sql_validation, - analysis_selection=CompileAnalysisSelection(no_cache=no_cache), - cli_vars=cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - - -def _read_target_name(*, discovered_inputs: DiscoveredProjectInputs, target_name: str) -> str: - target_config: TargetConfig = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ) - return target_config.defer_sources_to or target_name diff --git a/src/sqlbuild/compiler/pipeline/main/_clone.py b/src/sqlbuild/compiler/pipeline/main/_clone.py deleted file mode 100644 index 61bfa037b1..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/_clone.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Public clone pipeline entrypoint.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.clone import prepare_clone_pipeline -from sqlbuild.compiler.pipeline.models import ( - ClonePipelineConnection, - ClonePipelineOptions, - ClonePipelineResult, -) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver - - -def run_clone_pipeline( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - origin_target_name: str, - destination_target_name: str, - no_sql_validation: bool = False, - select: tuple[str, ...] = (), - exclude: tuple[str, ...] = (), - cli_vars: dict[str, object] | None = None, - destination_connection: ClonePipelineConnection, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, -) -> ClonePipelineResult: - return prepare_clone_pipeline( - discovered_inputs=discovered_inputs, - adapter=adapter, - origin_target_name=origin_target_name, - destination_target_name=destination_target_name, - destination_connection=destination_connection, - external_sql_reference_resolver=external_sql_reference_resolver, - options=ClonePipelineOptions( - no_sql_validation=no_sql_validation, - select=select, - exclude=exclude, - cli_vars=cli_vars, - ), - ) diff --git a/src/sqlbuild/compiler/pipeline/main/_static_command.py b/src/sqlbuild/compiler/pipeline/main/_static_command.py deleted file mode 100644 index b6bea2c6d4..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/_static_command.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Public static command compilation entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild._native import ( - build_deferred_locations, - compile_project_phase, - resolve_deferred_target_config, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.analysis_selection import apply_unit_test_selectors -from sqlbuild.compiler.pipeline.models import ( - CompiledProjectPhaseResult, - CompilePipelineOptions, - StaticCommandContext, -) -from sqlbuild.compiler.planner.main.commands._relations import resolve_static_relation_context -from sqlbuild.compiler.planner.main.commands._scope import resolve_static_command_scope -from sqlbuild.compiler.planner.models import ( - DeferralInputs, - PlannerRelationsContext, - PlannerScope, - PlannerSelection, -) -from sqlbuild.spec.contracts.models import TargetConfig - - -def compile_static_command_context( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - options: CompilePipelineOptions, - selected_keys: frozenset[CompiledObjectKey] | None = None, - relation_keys: frozenset[CompiledObjectKey] | None = None, - on_progress: Callable[[str], None] | None = None, -) -> StaticCommandContext: - """Run canonical compile, selector, and relation phases without build planning.""" - - options = apply_unit_test_selectors(options=options, discovered_inputs=discovered_inputs) - if selected_keys is None and not options.sql_test_selection.models_selected: - selected_keys = frozenset() - compiled: CompiledProjectPhaseResult = compile_project_phase( - discovered_inputs=discovered_inputs, - adapter=adapter, - options=options, - on_progress=on_progress, - ) - deferred_locations: dict[str, CompiledRelationLocation] | None = None - if options.defer_to is not None: - deferred_target: TargetConfig = resolve_deferred_target_config( - discovered_inputs=discovered_inputs, - defer_to=options.defer_to, - current_target_name=compiled.project.effective_target_name, - ) - deferred_locations = build_deferred_locations( - project=compiled.project, - deferred_target_config=deferred_target, - effective_vars=compiled.project.effective_vars, - default_schema=adapter.default_schema(), - default_database=adapter.default_database(), - render_qualified_name=adapter.render_qualified_name, - ) - scope: PlannerScope = resolve_static_command_scope( - project=compiled.project, - selection=PlannerSelection( - select=options.select, - exclude=options.exclude, - selected_keys=selected_keys, - sql_test_selection=options.sql_test_selection, - ), - auto_load_sources=options.auto_load_sources, - ) - relations: PlannerRelationsContext = resolve_static_relation_context( - project=compiled.project, - adapter=adapter, - scope=scope, - deferral=DeferralInputs( - deferred_locations=deferred_locations, - defer_sources_to=options.defer_sources_to, - source_deferral_enabled=options.source_deferral_enabled, - ), - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - relation_keys=relation_keys, - ) - return StaticCommandContext( - project=compiled.project, - scope=scope, - relations=relations, - connection_config=compiled.connection_config, - compile_seconds=compiled.compile_seconds, - ) diff --git a/src/sqlbuild/compiler/pipeline/main/clone_with_options.py b/src/sqlbuild/compiler/pipeline/main/clone_with_options.py deleted file mode 100644 index 8d8ffaa098..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/clone_with_options.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Clone pipeline entrypoint with typed compilation options.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.clone import prepare_clone_pipeline -from sqlbuild.compiler.pipeline.models import ( - ClonePipelineConnection, - ClonePipelineOptions, - ClonePipelineResult, -) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver - - -def run_clone_pipeline_with_options( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - origin_target_name: str, - destination_target_name: str, - destination_connection: ClonePipelineConnection, - options: ClonePipelineOptions, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, -) -> ClonePipelineResult: - """Run a clone pipeline with explicit compilation controls.""" - - return prepare_clone_pipeline( - discovered_inputs=discovered_inputs, - adapter=adapter, - origin_target_name=origin_target_name, - destination_target_name=destination_target_name, - destination_connection=destination_connection, - options=options, - external_sql_reference_resolver=external_sql_reference_resolver, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/__init__.py b/src/sqlbuild/compiler/planner/_helpers/changes/__init__.py deleted file mode 100644 index cf71b4f966..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Change detection and backfill policy resolution.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/actions.py b/src/sqlbuild/compiler/planner/_helpers/changes/actions.py deleted file mode 100644 index 2e870519c2..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/actions.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Per-model action resolution from each model's own change.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ( - ChangeDetectionResult, - PlannerChangeResults, - PlannerResolvedActions, - PlannerScope, - ResolvedModelAction, -) - - -def resolve_model_actions( - *, - scope: PlannerScope, - changes: PlannerChangeResults, -) -> PlannerResolvedActions: - """Resolve each selected model's backfill from its own change and replay_on_change only.""" - - resolved: dict[str, ResolvedModelAction] = {} - key: CompiledObjectKey - for key in scope.execution_order: - if key not in scope.selected_keys or key.resource_type != CompiledResourceType.MODEL: - continue - change: ChangeDetectionResult | None = changes.models.get(key.name) - if key.name not in scope.models_by_name or change is None: - continue - resolved[key.name] = ResolvedModelAction(change=change, backfill=change.backfill) - return PlannerResolvedActions(models=resolved) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py deleted file mode 100644 index dd611be592..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py +++ /dev/null @@ -1,375 +0,0 @@ -"""Per-model change detection orchestration.""" - -from __future__ import annotations - -import logging -from collections.abc import Callable - -from sqlbuild._native import get_config_str -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - InferredColumn, -) -from sqlbuild.compiler.compile.types import ( - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.changes.metadata import ( - changed_local_function_names, - non_function_identity_metadata_payload, - recorded_declared_columns_hash, - version_identity_metadata_payload, - with_declared_columns_hash, - with_origin_cursor_input_names, -) -from sqlbuild.compiler.planner._helpers.changes.policy import ( - pick_more_aggressive, - resolve_replay_on_change, -) -from sqlbuild.compiler.planner._helpers.changes.query import detect_query_change -from sqlbuild.compiler.planner._helpers.changes.reference_renames import origin_reference_names -from sqlbuild.compiler.planner._helpers.changes.schema import detect_schema_changes -from sqlbuild.compiler.planner._helpers.identity.functions import ( - build_compiled_function_fingerprint_sql, - detect_function_change, -) -from sqlbuild.compiler.planner._helpers.identity.hashing import model_definition_hash -from sqlbuild.compiler.planner._helpers.identity.model_metadata import declared_columns_hash -from sqlbuild.compiler.planner.constants import EMPTY_FINGERPRINT_METADATA_JSON -from sqlbuild.compiler.planner.main.identity.version_identity_function_hashes import ( - build_function_local_hashes, -) -from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( - build_model_version_identity_metadata_json, -) -from sqlbuild.compiler.planner.models import ( - BackfillResult, - ChangeDetectionResult, - FunctionChangeResult, - PlannerChangeResults, - PlannerScope, - SchemaFinding, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind -from sqlbuild.compiler.python_nodes.main._hook_identities import build_hook_identities -from sqlbuild.diagnostics.main.log_debug_event import log_debug_event -from sqlbuild.diagnostics.main.log_sql import log_sql - - -def detect_changes( - *, - project: CompiledProjectView, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - full_refresh: bool, - expected_version_hashes: dict[str, str] | None = None, - expected_metadata_jsons: dict[str, str] | None = None, -) -> PlannerChangeResults: - """Detect selected model and function changes.""" - - model_changes: dict[str, ChangeDetectionResult] = detect_model_changes_in_scope( - project=project, - scope=scope, - snapshot=snapshot, - expected_version_hashes=expected_version_hashes, - expected_metadata_jsons=expected_metadata_jsons, - ) - - function_changes: dict[str, FunctionChangeResult] = {} - function: CompiledFunctionView - for function in project.functions: - fingerprint_sql: str = build_compiled_function_fingerprint_sql(function) - if function.key not in scope.selected_keys: - function_changes[function.name] = FunctionChangeResult( - fingerprint_sql=fingerprint_sql, - ) - continue - function_changes[function.name] = FunctionChangeResult( - fingerprint_sql=fingerprint_sql, - reason=detect_function_change( - function=function, - fingerprint_sql=fingerprint_sql, - fingerprint=snapshot.fingerprints.functions.get(function.name), - query_change_tracking=project.settings.query_change_tracking, - full_refresh=full_refresh, - dialect=snapshot.column_dialect, - ), - ) - - return PlannerChangeResults(models=model_changes, functions=function_changes) - - -def detect_model_changes_in_scope( - *, - project: CompiledProjectView, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - query_change_tracking: bool | None = None, - expected_version_hashes: dict[str, str] | None = None, - expected_metadata_jsons: dict[str, str] | None = None, -) -> dict[str, ChangeDetectionResult]: - """Detect changes for selected models without inspecting functions.""" - - model_changes: dict[str, ChangeDetectionResult] = {} - effective_query_change_tracking: bool = ( - project.settings.query_change_tracking - if query_change_tracking is None - else query_change_tracking - ) - function_local_hashes: dict[str, str] = build_function_local_hashes( - functions=project.functions, dialect=project.sql_analysis_dialect - ) - hook_version_hashes: dict[str, str] = { - name: identity.version_hash - for name, identity in build_hook_identities(project.hook_functions).items() - } - key: CompiledObjectKey - for key in scope.execution_order: - if key not in scope.selected_keys or key.resource_type != CompiledResourceType.MODEL: - continue - model: CompiledModelView | None = scope.models_by_name.get(key.name) - if model is None: - continue - model_changes[model.name] = detect_model_changes( - model=model, - snapshot=snapshot, - sql_analysis_enabled=project.settings.sql_analysis, - query_change_tracking=effective_query_change_tracking, - full_refresh=False, - function_local_hashes=function_local_hashes, - hook_version_hashes=hook_version_hashes, - expected_version_hash=(expected_version_hashes or {}).get(model.name), - expected_metadata_json=(expected_metadata_jsons or {}).get(model.name), - ) - return model_changes - - -def detect_model_changes( - *, - model: CompiledModelView, - snapshot: WarehouseSnapshot, - sql_analysis_enabled: bool, - query_change_tracking: bool, - full_refresh: bool, - function_local_hashes: dict[str, str] | None = None, - hook_version_hashes: dict[str, str] | None = None, - expected_version_hash: str | None = None, - expected_metadata_json: str | None = None, -) -> ChangeDetectionResult: - """Detect changes for one model and resolve backfill policy.""" - - model_name: str = model.name - current_columns_hash: str = declared_columns_hash(model=model) - metadata_json: str = with_declared_columns_hash( - metadata_json=expected_metadata_json - or build_model_version_identity_metadata_json( - model=model, - function_local_hashes=function_local_hashes, - hook_version_hashes=hook_version_hashes, - ), - declared_columns_hash=current_columns_hash, - ) - - if full_refresh: - return ChangeDetectionResult( - model_name=model_name, - change_kind=ChangeKind.NO_CHANGE, - backfill=BackfillResult(action=BackfillAction.FULL), - fingerprint_metadata_json=metadata_json, - fingerprint_version_hash=expected_version_hash, - ) - - relation_exists: bool = model_name in snapshot.existing_relations - fingerprint: Fingerprint | None = snapshot.fingerprints.models.get(model_name) - compiled_query_hash: str | None = ( - model_definition_hash( - model_name=model_name, query_sql=model.query_sql, dialect=snapshot.column_dialect - ) - if query_change_tracking and fingerprint is not None - else None - ) - - if not relation_exists and model_name in snapshot.renamed_models: - renamed_query_changed: bool = ( - compiled_query_hash is not None - and fingerprint is not None - and detect_query_change( - compiled_query_hash=compiled_query_hash, fingerprint=fingerprint - ) - ) - return ChangeDetectionResult( - model_name=model_name, - change_kind=ChangeKind.RENAMED, - query_changed=renamed_query_changed, - backfill=( - resolve_replay_on_change( - replay_on_change=get_config_str( - values=model.config.values, key="replay_on_change" - ) - ) - if renamed_query_changed - else BackfillResult(action=BackfillAction.FORWARD_ONLY) - ), - fingerprint_metadata_json=metadata_json, - previous_metadata_json=fingerprint.metadata_json if fingerprint is not None else None, - fingerprint_version_hash=expected_version_hash, - previous_version_hash=fingerprint.version_hash if fingerprint is not None else None, - ) - if not relation_exists: - return ChangeDetectionResult( - model_name=model_name, - change_kind=ChangeKind.FIRST_RUN, - backfill=BackfillResult(action=BackfillAction.FULL), - fingerprint_metadata_json=metadata_json, - fingerprint_version_hash=expected_version_hash, - recorded_build_relation_missing=fingerprint is not None, - ) - - query_changed: bool = False - recorded_metadata_json: str | None = ( - fingerprint.metadata_json - if fingerprint is not None and fingerprint.metadata_json != EMPTY_FINGERPRINT_METADATA_JSON - else None - ) - changed_functions: tuple[str, ...] = ( - changed_local_function_names( - metadata_json=metadata_json, previous_metadata_json=recorded_metadata_json - ) - if query_change_tracking and recorded_metadata_json is not None - else () - ) - identity_payload: Callable[[str | None], object] = ( - non_function_identity_metadata_payload - if changed_functions - else version_identity_metadata_payload - ) - origin_names: dict[str, str] | None = ( - origin_reference_names( - query_sql=model.query_sql, - recorded=fingerprint, - renames=snapshot.reference_renames, - dialect=snapshot.column_dialect, - ) - if compiled_query_hash is not None - and fingerprint is not None - and snapshot.reference_renames - and compiled_query_hash != fingerprint.definition_hash - else None - ) - config_changed: bool = recorded_metadata_json is not None and identity_payload( - with_origin_cursor_input_names(metadata_json=metadata_json, renamed_refs=origin_names) - if origin_names is not None - else metadata_json - ) != identity_payload(recorded_metadata_json) - replay_backfill: BackfillResult = BackfillResult(action=BackfillAction.FORWARD_ONLY) - if compiled_query_hash is not None and fingerprint is not None: - debug_logger: logging.Logger = logging.getLogger("sqlbuild.planner.changes") - query_changed = origin_names is None and detect_query_change( - compiled_query_hash=compiled_query_hash, - fingerprint=fingerprint, - ) - log_debug_event( - logger=debug_logger, - message=( - "fingerprint comparison" - f" compiled_query_hash={compiled_query_hash}" - f" fingerprint_definition_hash={fingerprint.definition_hash}" - f" query_changed={query_changed}" - ), - sqlbuild_subject="model", - sqlbuild_name=model_name, - sqlbuild_event="query_change_check", - sqlbuild_phase="planner", - sqlbuild_status="changed" if query_changed else "unchanged", - ) - log_sql(logger=debug_logger, sql=model.query_sql, action="compiled_query") - log_sql(logger=debug_logger, sql=fingerprint.definition, action="fingerprint_definition") - if query_changed or changed_functions: - replay_backfill = resolve_replay_on_change( - replay_on_change=get_config_str(values=model.config.values, key="replay_on_change") - ) - - schema_findings: tuple[SchemaFinding, ...] = () - schema_backfill: BackfillResult = BackfillResult(action=BackfillAction.FORWARD_ONLY) - warehouse_columns: tuple[ColumnInfo, ...] | None = snapshot.existing_columns.get(model_name) - yml_columns: tuple[ColumnInfo, ...] = _build_yml_columns(model) - inferred_columns: tuple[InferredColumn, ...] | None = model.inferred_columns - has_expected: bool = bool(yml_columns) or ( - inferred_columns is not None and bool(inferred_columns) - ) - type_enforcement: bool = _get_type_enforcement(model) - if warehouse_columns is not None and has_expected: - schema_findings = detect_schema_changes( - yml_columns=yml_columns, - inferred_columns=inferred_columns, - warehouse_columns=warehouse_columns, - type_enforcement=type_enforcement, - inferred_schema_complete=not model.fast_lineage_has_star, - dynamic_columns=( - model.schema_entry.dynamic_columns if model.schema_entry is not None else () - ), - dialect=snapshot.column_dialect, - ) - recorded_columns_hash: str | None = recorded_declared_columns_hash(recorded_metadata_json) - declared_columns_changed: bool = ( - recorded_columns_hash is not None and recorded_columns_hash != current_columns_hash - ) - if schema_findings and ( - query_changed or changed_functions or config_changed or declared_columns_changed - ): - schema_backfill = resolve_replay_on_change( - replay_on_change=get_config_str(values=model.config.values, key="replay_on_change") - ) - - backfill: BackfillResult = pick_more_aggressive(a=replay_backfill, b=schema_backfill) - - change_kind: ChangeKind - if query_changed: - change_kind = ChangeKind.QUERY_CHANGED - elif changed_functions: - change_kind = ChangeKind.FUNCTION_CHANGED - elif schema_findings: - change_kind = ChangeKind.SCHEMA_CHANGED - elif config_changed: - change_kind = ChangeKind.CONFIG_CHANGED - else: - change_kind = ChangeKind.NO_CHANGE - - return ChangeDetectionResult( - model_name=model_name, - change_kind=change_kind, - query_changed=query_changed, - config_changed=config_changed, - fingerprint_metadata_json=metadata_json, - previous_metadata_json=fingerprint.metadata_json if fingerprint is not None else None, - fingerprint_version_hash=expected_version_hash, - previous_version_hash=fingerprint.version_hash if fingerprint is not None else None, - schema_findings=schema_findings, - backfill=backfill, - changed_functions=changed_functions, - ) - - -def _get_type_enforcement(model: CompiledModelView) -> bool: - """Resolve whether type enforcement is active for a model.""" - - if model.schema_entry is not None and model.schema_entry.type_enforcement is not None: - return model.schema_entry.type_enforcement - return False - - -def _build_yml_columns(model: CompiledModelView) -> tuple[ColumnInfo, ...]: - """Build expected columns from schema.yml declarations.""" - - if model.schema_entry is None: - return () - return tuple( - ColumnInfo(name=col.name, type=col.type) - for col in model.schema_entry.columns - if col.type is not None - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py b/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py deleted file mode 100644 index bbc84be95a..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Version identity metadata comparison helpers.""" - -from __future__ import annotations - -import json - -from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY -from sqlbuild.compiler.fingerprints.constants import AUDIT_GATE_METADATA_KEY -from sqlbuild.compiler.planner.constants import ( - DECLARED_COLUMNS_METADATA_KEY, - LOCAL_FUNCTION_HASHES_METADATA_KEY, - MIGRATION_FINGERPRINT_METADATA_KEY, -) - - -def version_identity_metadata_payload(metadata_json: str | None) -> object: - """Return the fingerprint metadata payload that participates in version identity.""" - - if metadata_json is None: - return None - try: - payload: object = json.loads(metadata_json) - except json.JSONDecodeError: - return metadata_json - if not isinstance(payload, dict): - return None - identity_payload: dict[str, object] = dict(payload) - identity_payload.pop(AUDIT_GATE_METADATA_KEY, None) - identity_payload.pop(MIGRATION_FINGERPRINT_METADATA_KEY, None) - identity_payload.pop(DECLARED_COLUMNS_METADATA_KEY, None) - return identity_payload - - -def non_function_identity_metadata_payload(metadata_json: str | None) -> object: - """Return the version identity metadata payload without called-function hashes.""" - - payload: object = version_identity_metadata_payload(metadata_json) - if not isinstance(payload, dict): - return payload - return { - key: value for key, value in payload.items() if key != LOCAL_FUNCTION_HASHES_METADATA_KEY - } - - -def changed_local_function_names( - *, metadata_json: str, previous_metadata_json: str -) -> tuple[str, ...]: - """Return directly called functions whose recorded definition hash changed.""" - - current: dict[str, object] = _local_function_hashes(metadata_json) - previous: dict[str, object] = _local_function_hashes(previous_metadata_json) - return tuple( - sorted( - name - for name, function_hash in current.items() - if name in previous and previous[name] != function_hash - ) - ) - - -def _local_function_hashes(metadata_json: str) -> dict[str, object]: - try: - payload: object = json.loads(metadata_json) - except json.JSONDecodeError: - return {} - if not isinstance(payload, dict): - return {} - hashes: object = payload.get(LOCAL_FUNCTION_HASHES_METADATA_KEY) - return {str(name): value for name, value in hashes.items()} if isinstance(hashes, dict) else {} - - -def with_origin_cursor_input_names(*, metadata_json: str, renamed_refs: dict[str, str]) -> str: - """Return metadata JSON whose cursor_inputs name renamed models by their origin names.""" - - try: - payload: object = json.loads(metadata_json) - except json.JSONDecodeError: - return metadata_json - config: object = payload.get("config") if isinstance(payload, dict) else None - cursor_inputs: object = ( - config.get(CURSOR_INPUTS_CONFIG_KEY) if isinstance(config, dict) else None - ) - if ( - not isinstance(payload, dict) - or not isinstance(config, dict) - or not isinstance(cursor_inputs, dict) - ): - return metadata_json - origin_inputs: dict[str, object] = { - renamed_refs.get(str(name), str(name)): value for name, value in cursor_inputs.items() - } - return json.dumps( - {**payload, "config": {**config, CURSOR_INPUTS_CONFIG_KEY: origin_inputs}}, - sort_keys=True, - separators=(",", ":"), - default=str, - ) - - -def recorded_declared_columns_hash(metadata_json: str | None) -> str | None: - """Return the declared-columns hash a fingerprint row recorded, if it has one.""" - - if metadata_json is None: - return None - try: - payload: object = json.loads(metadata_json) - except json.JSONDecodeError: - return None - value: object = ( - payload.get(DECLARED_COLUMNS_METADATA_KEY) if isinstance(payload, dict) else None - ) - return value if isinstance(value, str) else None - - -def with_declared_columns_hash(*, metadata_json: str, declared_columns_hash: str) -> str: - """Return fingerprint metadata JSON that also records the model's declared-columns hash.""" - - try: - payload: object = json.loads(metadata_json) - except json.JSONDecodeError: - return metadata_json - if not isinstance(payload, dict): - return metadata_json - return json.dumps( - {**payload, DECLARED_COLUMNS_METADATA_KEY: declared_columns_hash}, - sort_keys=True, - separators=(",", ":"), - default=str, - ) - - -def without_declared_columns_hash(metadata_json: str) -> str: - """Return fingerprint metadata JSON without the declared-columns hash key, if present.""" - - try: - payload: object = json.loads(metadata_json) - except json.JSONDecodeError: - return metadata_json - if not isinstance(payload, dict) or DECLARED_COLUMNS_METADATA_KEY not in payload: - return metadata_json - return json.dumps( - {key: value for key, value in payload.items() if key != DECLARED_COLUMNS_METADATA_KEY}, - sort_keys=True, - separators=(",", ":"), - default=str, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/policy.py b/src/sqlbuild/compiler/planner/_helpers/changes/policy.py deleted file mode 100644 index 2195b5de55..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/policy.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Replay-on-change policy resolution.""" - -from __future__ import annotations - -import re - -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import BackfillResult -from sqlbuild.compiler.planner.types import BackfillAction - -_BOUNDED_PATTERN: re.Pattern[str] = re.compile(r"^bounded-(.+)$") - -_ACTION_PRIORITY: dict[BackfillAction, int] = { - BackfillAction.FORWARD_ONLY: 0, - BackfillAction.BOUNDED: 1, - BackfillAction.FULL: 2, -} - - -def resolve_replay_on_change( - *, - replay_on_change: str | None, -) -> BackfillResult: - """Resolve replay behavior for a detected model version change.""" - - return _resolve_replay_value(replay_on_change) - - -def pick_more_aggressive(*, a: BackfillResult, b: BackfillResult) -> BackfillResult: - """Return the more aggressive of two backfill results.""" - - if _ACTION_PRIORITY[a.action] >= _ACTION_PRIORITY[b.action]: - return a - return b - - -def _resolve_replay_value(raw: str | None) -> BackfillResult: - """Parse a raw replay policy value into a BackfillResult.""" - - if raw is None or raw == BackfillAction.FORWARD_ONLY: - return BackfillResult(action=BackfillAction.FORWARD_ONLY) - if raw == BackfillAction.FULL: - return BackfillResult(action=BackfillAction.FULL) - match: re.Match[str] | None = _BOUNDED_PATTERN.match(raw) - if match is not None: - duration: str = match.group(1).strip() - return BackfillResult(action=BackfillAction.BOUNDED, duration=duration) - raise PlannerInputError(f"unknown replay_on_change '{raw}'") diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/query.py b/src/sqlbuild/compiler/planner/_helpers/changes/query.py deleted file mode 100644 index 5cbbb0320f..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/query.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Query change detection by hash comparison.""" - -from __future__ import annotations - -from sqlbuild.compiler.fingerprints.models import Fingerprint - - -def detect_query_change( - *, - compiled_query_hash: str, - fingerprint: Fingerprint, -) -> bool: - """Compare compiled hashes against the stored fingerprint to detect query changes.""" - - return compiled_query_hash != fingerprint.definition_hash diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py b/src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py deleted file mode 100644 index d0d7336477..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Collect the model renames that downstream references may have been rewritten for.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.identity.hashing import model_definition_hash -from sqlbuild.compiler.planner.classes.migration_state_inspection import ( - MigrationStateInspection, -) -from sqlbuild.compiler.planner.models import ( - PlannerRuntime, - PlannerScope, - ReferenceRename, - WarehouseSnapshot, -) -from sqlbuild.compiler.references.types import SqlReferenceKind - - -def plan_reference_renames( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - state: MigrationStateInspection, - schemas: set[str], - planned: dict[str, str], -) -> tuple[ReferenceRename, ...]: - """Return this plan's renames plus recorded renames, read only when a query changed.""" - - if not _has_changed_referencing_query(scope=scope, snapshot=snapshot): - return () - state.inspect_migration_events( - schemas={schema for schema in schemas if schema.lower() in snapshot.migration_state_schemas} - ) - target_name: str | None = runtime.project.effective_target_name - return ( - *( - ReferenceRename(new_name=new_name, origin_name=origin_name) - for new_name, origin_name in planned.items() - if new_name != origin_name - ), - *( - ReferenceRename( - new_name=event.destination_model, - origin_name=event.origin_model, - recorded_at=event.created_at, - ) - for event in state.events - if event.target_name in (None, target_name) - and event.origin_model is not None - and event.origin_model != event.destination_model - ), - ) - - -def _has_changed_referencing_query(*, scope: PlannerScope, snapshot: WarehouseSnapshot) -> bool: - selected: tuple[CompiledModelView, ...] = tuple( - model - for key in scope.selected_keys - if (model := scope.models_by_name.get(key.name)) is not None - ) - return any(_referencing_query_changed(model=model, snapshot=snapshot) for model in selected) - - -def _referencing_query_changed(*, model: CompiledModelView, snapshot: WarehouseSnapshot) -> bool: - recorded: Fingerprint | None = snapshot.fingerprints.models.get(model.name) - return ( - recorded is not None - and any(reference.ref_kind == SqlReferenceKind.REF for reference in model.references) - and model_definition_hash( - model_name=model.name, query_sql=model.query_sql, dialect=snapshot.column_dialect - ) - != recorded.definition_hash - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py b/src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py deleted file mode 100644 index 6560c5b9bd..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Recognize model queries whose only change is references rewritten for renamed models.""" - -from __future__ import annotations - -import re -from datetime import UTC, datetime -from itertools import islice, product - -from sqlbuild._native import compute_query_hash, neutralize_lint_interpolation -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import ReferenceRename - -_REF_CALL: re.Pattern[str] = re.compile( - r"^__ref\s*\(\s*(?P['\"])(?P[^'\"]+)(?P=quote)\s*\)$" -) -_GENERIC_DIALECT: str = "generic" -_MAX_CANDIDATE_MAPPINGS: int = 64 - -type _NameSpan = tuple[int, int, str] - - -def origin_reference_names( - *, - query_sql: str, - recorded: Fingerprint, - renames: tuple[ReferenceRename, ...], - dialect: str | None, -) -> dict[str, str] | None: - """Return the rename mapping under which the query hashes to its recorded definition.""" - - origins: dict[str, list[str]] = {} - rename: ReferenceRename - for rename in renames: - if rename.recorded_at is None or _utc(recorded.ts) < _utc(rename.recorded_at): - origins.setdefault(rename.new_name, []).append(rename.origin_name) - if not origins: - return None - spans: tuple[_NameSpan, ...] = _ref_name_spans( - query_sql=query_sql, dialect=dialect or _GENERIC_DIALECT, names=frozenset(origins) - ) - names: tuple[str, ...] = tuple(sorted({name for _, _, name in spans})) - choices: tuple[tuple[str, ...], ...] = tuple( - _lineage(name=name, origins=origins) for name in names - ) - combination: tuple[str, ...] - for combination in islice(product(*choices), _MAX_CANDIDATE_MAPPINGS): - mapping: dict[str, str] = { - name: origin for name, origin in zip(names, combination, strict=True) if origin != name - } - if mapping and compute_query_hash( - query_sql=_substitute(query_sql=query_sql, spans=spans, mapping=mapping), - dialect=dialect, - ) == (recorded.definition_hash): - return mapping - return None - - -def _ref_name_spans( - *, query_sql: str, dialect: str, names: frozenset[str] -) -> tuple[_NameSpan, ...]: - spans: list[_NameSpan] = [] - original_start: int - original_text: str - for _, _, _, original_start, _, original_text in neutralize_lint_interpolation( - query_sql, dialect - )[1]: - match: re.Match[str] | None = _REF_CALL.match(original_text) - if match is not None and match.group("name") in names: - spans.append( - ( - original_start + match.start("name"), - original_start + match.end("name"), - match.group("name"), - ) - ) - return tuple(spans) - - -def _lineage(*, name: str, origins: dict[str, list[str]]) -> tuple[str, ...]: - seen: list[str] = [name] - index: int = 0 - while index < len(seen): - origin: str - for origin in origins.get(seen[index], ()): - if origin not in seen: - seen.append(origin) - index += 1 - return tuple(seen) - - -def _substitute(*, query_sql: str, spans: tuple[_NameSpan, ...], mapping: dict[str, str]) -> str: - pieces: list[str] = [] - copied_to: int = 0 - start: int - end: int - name: str - for start, end, name in spans: - pieces.extend((query_sql[copied_to:start], mapping.get(name, name))) - copied_to = end - pieces.append(query_sql[copied_to:]) - return "".join(pieces) - - -def _utc(value: datetime) -> datetime: - if value.tzinfo is None or value.utcoffset() is None: - return value.replace(tzinfo=UTC) - return value.astimezone(UTC) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/schema.py b/src/sqlbuild/compiler/planner/_helpers/changes/schema.py deleted file mode 100644 index 6da7f7e8f8..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/schema.py +++ /dev/null @@ -1,237 +0,0 @@ -"""Schema change detection by column comparison.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.compile.models import InferredColumn -from sqlbuild.compiler.planner.models import SchemaFinding -from sqlbuild.compiler.planner.types import SchemaChangeKind, SchemaColumnSource -from sqlbuild.spec.contracts.main.matching_dynamic_families import matching_dynamic_families -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - -_CASE_FOLDING_DIALECT: str = "snowflake" - - -def detect_schema_changes( - *, - yml_columns: tuple[ColumnInfo, ...], - inferred_columns: tuple[InferredColumn, ...] | None, - warehouse_columns: tuple[ColumnInfo, ...], - type_enforcement: bool, - inferred_schema_complete: bool, - dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = (), - dialect: str | None = None, -) -> tuple[SchemaFinding, ...]: - """Compare yml and inferred columns against warehouse columns and return findings.""" - - if dialect == _CASE_FOLDING_DIALECT: - yml_columns = tuple(replace(col, name=col.name.lower()) for col in yml_columns) - if inferred_columns is not None: - inferred_columns = tuple( - replace(col, name=col.name.lower()) for col in inferred_columns - ) - warehouse_columns = tuple(replace(col, name=col.name.lower()) for col in warehouse_columns) - warehouse_map: dict[str, str] = {col.name: col.type for col in warehouse_columns} - findings: list[SchemaFinding] = [] - - seen_names: set[str] = set() - - if type_enforcement: - findings.extend( - _compare_yml_columns( - yml_columns=yml_columns, warehouse_map=warehouse_map, dialect=dialect - ) - ) - col: ColumnInfo - for col in yml_columns: - seen_names.add(col.name) - - if inferred_columns is not None: - findings.extend( - _compare_inferred_columns( - inferred_columns=inferred_columns, - warehouse_map=warehouse_map, - seen_names=seen_names, - dialect=dialect, - ) - ) - inferred_col: InferredColumn - for inferred_col in inferred_columns: - seen_names.add(inferred_col.name) - else: - if inferred_columns is not None: - findings.extend( - _compare_inferred_columns( - inferred_columns=inferred_columns, - warehouse_map=warehouse_map, - seen_names=seen_names, - dialect=dialect, - ) - ) - inferred_col_ne: InferredColumn - for inferred_col_ne in inferred_columns: - seen_names.add(inferred_col_ne.name) - - findings.extend( - _compare_yml_columns_non_enforced( - yml_columns=yml_columns, - warehouse_map=warehouse_map, - seen_names=seen_names, - dialect=dialect, - ) - ) - col_ne: ColumnInfo - for col_ne in yml_columns: - seen_names.add(col_ne.name) - - for column_name, column_type in warehouse_map.items(): - if column_name in seen_names: - continue - matching: tuple[SchemaDynamicColumnFamily, ...] = matching_dynamic_families( - families=dynamic_columns, - column_name=column_name, - ) - if len(matching) != 1: - continue - family: SchemaDynamicColumnFamily = matching[0] - seen_names.add(column_name) - if not types_equal(left=family.type, right=column_type, dialect=dialect): - findings.append( - SchemaFinding( - kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, - column_name=column_name, - source=SchemaColumnSource.YML, - expected_type=family.type, - actual_type=column_type, - ) - ) - - if type_enforcement or inferred_schema_complete: - col_name: str - col_type: str - for col_name, col_type in warehouse_map.items(): - if col_name not in seen_names: - findings.append( - SchemaFinding( - kind=SchemaChangeKind.COLUMN_REMOVED, - column_name=col_name, - source=( - SchemaColumnSource.YML - if yml_columns - else SchemaColumnSource.SQL_ANALYSIS - ), - actual_type=col_type, - ) - ) - - return tuple(findings) - - -def _compare_yml_columns( - *, - yml_columns: tuple[ColumnInfo, ...], - warehouse_map: dict[str, str], - dialect: str | None, -) -> list[SchemaFinding]: - """Compare yml-declared columns against warehouse state.""" - - findings: list[SchemaFinding] = [] - col: ColumnInfo - for col in yml_columns: - if col.name not in warehouse_map: - findings.append( - SchemaFinding( - kind=SchemaChangeKind.COLUMN_ADDED, - column_name=col.name, - source=SchemaColumnSource.YML, - expected_type=col.type, - ) - ) - elif not types_equal(left=warehouse_map[col.name], right=col.type, dialect=dialect): - findings.append( - SchemaFinding( - kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, - column_name=col.name, - source=SchemaColumnSource.YML, - expected_type=col.type, - actual_type=warehouse_map[col.name], - ) - ) - return findings - - -def _compare_yml_columns_non_enforced( - *, - yml_columns: tuple[ColumnInfo, ...], - warehouse_map: dict[str, str], - seen_names: set[str], - dialect: str | None, -) -> list[SchemaFinding]: - """Compare yml columns against warehouse when type enforcement is off.""" - - findings: list[SchemaFinding] = [] - col: ColumnInfo - for col in yml_columns: - if col.name in seen_names: - continue - if col.name not in warehouse_map: - findings.append( - SchemaFinding( - kind=SchemaChangeKind.COLUMN_ADDED, - column_name=col.name, - source=SchemaColumnSource.YML, - expected_type=col.type, - ) - ) - elif not types_equal(left=warehouse_map[col.name], right=col.type, dialect=dialect): - findings.append( - SchemaFinding( - kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, - column_name=col.name, - source=SchemaColumnSource.YML, - expected_type=col.type, - actual_type=warehouse_map[col.name], - ) - ) - return findings - - -def _compare_inferred_columns( - *, - inferred_columns: tuple[InferredColumn, ...], - warehouse_map: dict[str, str], - seen_names: set[str], - dialect: str | None, -) -> list[SchemaFinding]: - """Compare sql_analysis-inferred columns against warehouse state, skipping yml-covered names.""" - - findings: list[SchemaFinding] = [] - col: InferredColumn - for col in inferred_columns: - if col.name in seen_names: - continue - if col.name not in warehouse_map: - findings.append( - SchemaFinding( - kind=SchemaChangeKind.COLUMN_ADDED, - column_name=col.name, - source=SchemaColumnSource.SQL_ANALYSIS, - expected_type=col.type, - ) - ) - elif col.type is not None and not types_equal( - left=warehouse_map[col.name], right=col.type, dialect=dialect - ): - findings.append( - SchemaFinding( - kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, - column_name=col.name, - source=SchemaColumnSource.SQL_ANALYSIS, - expected_type=col.type, - actual_type=warehouse_map[col.name], - ) - ) - return findings diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py index a325947579..fa198254ff 100644 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py +++ b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import get_materialization_type from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView @@ -20,7 +21,6 @@ plan_function, sql_test_is_selected, ) -from sqlbuild.compiler.planner._helpers.output.strategy import get_materialization_type from sqlbuild.compiler.planner._helpers.planning.scopes import resolve_planner_scopes from sqlbuild.compiler.planner.models import ( AuditPlanEntry, diff --git a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py b/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py index 139e393c53..1f1389a977 100644 --- a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py +++ b/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py @@ -8,10 +8,8 @@ from functools import lru_cache from typing import Any +from sqlbuild._native import analyze_resolved_column_reads from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.main._analyze_columns_and_lineage import ( - analyze_resolved_column_reads, -) from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, CompiledLineageSourceFact, diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py b/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py index 086cd9a8fe..f921cf634e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py +++ b/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py @@ -2,10 +2,10 @@ from __future__ import annotations +from sqlbuild._native import loader_destination_parts from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.spec.contracts.main.loader_destination_parts import loader_destination_parts from sqlbuild.spec.contracts.models import LoaderDestinationParts, SourceEntry diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py b/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py index 548567bdc4..b60b5bbbc1 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py @@ -2,13 +2,12 @@ from __future__ import annotations -from sqlbuild._native import get_config_str +from sqlbuild._native import detect_model_changes, get_config_str from sqlbuild.adapter.contract.models import ColumnInfo, NormalizedType, RelationInfo from sqlbuild.adapter.contract.types import TypeFamily from sqlbuild.adapter.type_system.main.normalize_type import normalize_type from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.migrations.types import MigrationCompatibility -from sqlbuild.compiler.planner._helpers.changes.detect import detect_model_changes from sqlbuild.compiler.planner.constants import VIEW_RELATION_TYPE_MARKER from sqlbuild.compiler.planner.models import ( ChangeDetectionResult, diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py b/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py index b41baeb9a2..31e74658f5 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py @@ -8,8 +8,8 @@ from collections.abc import Mapping from typing import Any +from sqlbuild._native import version_identity_metadata_payload from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY -from sqlbuild.compiler.planner._helpers.changes.metadata import version_identity_metadata_payload from sqlbuild.compiler.planner.constants import ( MIGRATION_FINGERPRINT_EXCLUDED_CONFIG_KEYS, MIGRATION_LOCAL_NAME_PREFIX, diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index f2cc964f6d..a0df66e45c 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -6,7 +6,7 @@ from dataclasses import replace from datetime import UTC, datetime -from sqlbuild._native import compute_query_hash, get_config_str +from sqlbuild._native import compute_query_hash, get_config_str, plan_reference_renames from sqlbuild.adapter.contract.models import ColumnInfo, MigrationStagePlan, RelationInfo from sqlbuild.adapter.contract.types import BuiltinAdapter, MigrationTransfer from sqlbuild.compiler.compile.constants import MIGRATE_FORCE_CONFIG_KEY, MIGRATE_FROM_CONFIG_KEY @@ -22,7 +22,6 @@ MigrationDiscovery, MigrationPromotion, ) -from sqlbuild.compiler.planner._helpers.changes.recorded_renames import plan_reference_renames from sqlbuild.compiler.planner._helpers.migrations.compatibility import ( MigrationCompatibilityResult, check_migration_compatibility, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py index dd78a3f630..93ec534f32 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py @@ -2,7 +2,13 @@ from __future__ import annotations -from sqlbuild._native import assert_no_unresolved_sql_markers, render_audit_sql +from sqlbuild._native import ( + assert_no_unresolved_sql_markers, + reads_outside_target_lineage, + render_audit_sql, + resolve_attachment_kind, + resolve_effective_run_scope, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, @@ -14,11 +20,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledAuditView -from sqlbuild.compiler.planner._helpers.output.audit_scheduling import ( - reads_outside_target_lineage, - resolve_attachment_kind, - resolve_effective_run_scope, -) from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.spec.contracts.models import SourceEntry diff --git a/src/sqlbuild/compiler/planner/_helpers/output/audit_scheduling.py b/src/sqlbuild/compiler/planner/_helpers/output/audit_scheduling.py deleted file mode 100644 index e4851055be..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/audit_scheduling.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Audit attachment resolution and scheduling validation.""" - -from __future__ import annotations - -from sqlbuild.compiler.auditing.types import ( - AuditAttachmentKind, - AuditRunScope, -) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, -) -from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, - CompiledAuditView, - CompiledResourceType, -) -from sqlbuild.compiler.planner._helpers.graph.core import expand_downstream, expand_upstream -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.types import MaterializationType -from sqlbuild.compiler.references.types import SqlReferenceKind - -_ATTACHMENT_KINDS: dict[AttachedAuditTargetKind, AuditAttachmentKind] = { - AttachedAuditTargetKind.MODEL: AuditAttachmentKind.MODEL, - AttachedAuditTargetKind.SOURCE: AuditAttachmentKind.SOURCE, - AttachedAuditTargetKind.SEED: AuditAttachmentKind.SEED, -} - - -def resolve_attachment_kind( - *, - audit: CompiledAuditView, - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> tuple[AuditAttachmentKind, str | None]: - """Resolve audit attachment kind and attached target name; attached audits gate their target.""" - - if audit.attached_target_kind is not None: - if audit.attached_target_name is None: - raise PlannerInputError( - f"audit '{audit.name}': attached audit is missing an attached target name" - ) - return _ATTACHMENT_KINDS[AttachedAuditTargetKind(audit.attached_target_kind)], ( - audit.attached_target_name - ) - return _infer_singular_attachment(audit=audit, downstream_deps=downstream_deps) - - -def reads_outside_target_lineage( - *, - audit: CompiledAuditView, - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> bool: - """Return whether an attached model audit reads anything besides its target and upstream.""" - - if audit.attached_target_kind is None or audit.attached_target_name is None: - return False - target: CompiledObjectKey = CompiledObjectKey( - resource_type=AttachedAuditTargetKind(audit.attached_target_kind).resource_type, - name=audit.attached_target_name, - ) - lineage: frozenset[CompiledObjectKey] = frozenset( - {target, *expand_upstream(key=target, upstream=upstream_deps)} - ) - return any(dep_key not in lineage for dep_key in audit.scope_deps) - - -def resolve_effective_run_scope( - *, - requested_run_scope: AuditRunScope, - attached_model_materialization: str | None, -) -> AuditRunScope: - """Resolve effective run scope after degradation rules.""" - - if requested_run_scope == AuditRunScope.FINAL: - return AuditRunScope.FINAL - - if attached_model_materialization not in ( - MaterializationType.INCREMENTAL, - MaterializationType.SNAPSHOT, - ): - return AuditRunScope.FINAL - - return AuditRunScope.DELTA_AND_FINAL - - -def _infer_singular_attachment( - *, - audit: CompiledAuditView, - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> tuple[AuditAttachmentKind, str | None]: - """Infer attachment for a singular audit from its refs and graph structure.""" - - model_ref_names: list[str] = [ - ref.ref_name for ref in audit.references if ref.ref_kind == SqlReferenceKind.REF - ] - source_ref_names: list[str] = [ - ref.ref_name for ref in audit.references if ref.ref_kind == SqlReferenceKind.SOURCE - ] - - if not model_ref_names and source_ref_names: - attached_source: str = sorted(source_ref_names)[0] - return AuditAttachmentKind.SOURCE, attached_source - - if not model_ref_names and not source_ref_names: - return AuditAttachmentKind.END, None - - if len(model_ref_names) == 1: - return AuditAttachmentKind.MODEL, model_ref_names[0] - - latest_owner: str | None = _find_single_safe_latest_owner( - model_ref_names=model_ref_names, - downstream_deps=downstream_deps, - ) - if latest_owner is not None: - return AuditAttachmentKind.MODEL, latest_owner - - return AuditAttachmentKind.END, None - - -def _find_single_safe_latest_owner( - *, - model_ref_names: list[str], - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> str | None: - """Find the single model that all other referenced models can reach transitively.""" - - model_keys: list[CompiledObjectKey] = [ - CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=name) - for name in model_ref_names - ] - model_key_set: frozenset[CompiledObjectKey] = frozenset(model_keys) - - candidates: list[CompiledObjectKey] = [] - candidate: CompiledObjectKey - for candidate in model_keys: - all_reach: bool = True - other: CompiledObjectKey - for other in model_keys: - if other == candidate: - continue - reachable: frozenset[CompiledObjectKey] = expand_downstream( - key=other, downstream=downstream_deps - ) - if candidate not in reachable: - all_reach = False - break - if all_reach: - candidates.append(candidate) - - if len(candidates) == 1: - return candidates[0].name - - if len(candidates) > 1: - ordered: list[CompiledObjectKey] = sorted(candidates, key=lambda k: k.name) - deepest: CompiledObjectKey = ordered[0] - other_candidate: CompiledObjectKey - for other_candidate in ordered[1:]: - reachable_from_deepest: frozenset[CompiledObjectKey] = expand_downstream( - key=deepest, downstream=downstream_deps - ) - if other_candidate in reachable_from_deepest: - deepest = other_candidate - all_others_reach: bool = True - check_key: CompiledObjectKey - for check_key in model_key_set: - if check_key == deepest: - continue - if deepest not in expand_downstream(key=check_key, downstream=downstream_deps): - all_others_reach = False - break - if all_others_reach: - return deepest.name - - return None diff --git a/src/sqlbuild/compiler/planner/_helpers/output/cursor_type_check.py b/src/sqlbuild/compiler/planner/_helpers/output/cursor_type_check.py deleted file mode 100644 index 419b9b4505..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/cursor_type_check.py +++ /dev/null @@ -1,219 +0,0 @@ -"""Warehouse cursor column type consistency checking.""" - -from __future__ import annotations - -import logging -import re -from typing import Any - -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.planner.constants import POLYGLOT_CUSTOM_DATA_TYPE_NAME -from sqlbuild.compiler.planner.models import PlanWarning -from sqlbuild.compiler.planner.types import CursorType, WarningSeverity -from sqlbuild.compiler.sql_analysis.main.import_polyglot import import_polyglot -from sqlbuild.diagnostics.main.log_debug_event import log_debug_event - -_DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.planner") - -_POLYGLOT_TIMESTAMP_TYPE_NAMES: frozenset[str] = frozenset( - { - "TIMESTAMP", - "TIMESTAMPNTZ", - "TIMESTAMPLTZ", - "TIMESTAMPTZ", - "TIMESTAMP_S", - "TIMESTAMP_MS", - "TIMESTAMP_NS", - "DATE", - "DATE32", - "DATETIME", - "DATETIME2", - "DATETIME64", - "SMALLDATETIME", - } -) - -_POLYGLOT_INTEGER_TYPE_NAMES: frozenset[str] = frozenset( - { - "INT", - "BIG_INT", - "BIGINT", - "SMALL_INT", - "SMALLINT", - "TINY_INT", - "TINYINT", - "MEDIUMINT", - "INT128", - "INT256", - "UINT", - "UBIGINT", - "USMALLINT", - "UTINYINT", - "UMEDIUMINT", - "UINT128", - "UINT256", - } -) -_DECIMAL_INTEGER_TYPE_PATTERN: re.Pattern[str] = re.compile( - r"^(?:DECIMAL|NUMERIC|NUMBER)(?:\(\s*\d+\s*(?:,\s*(\d+)\s*)?\))?$" -) - - -def check_cursor_type_consistency( - *, - model_name: str, - cursor_column: str | None, - cursor_type: str | None, - warehouse_columns: tuple[ColumnInfo, ...], - sql_analysis_enabled: bool, -) -> PlanWarning | None: - """Check whether the warehouse column type is consistent with cursor_type.""" - - if cursor_column is None or cursor_type is None: - return None - - warehouse_type: str | None = _find_column_type( - columns=warehouse_columns, column_name=cursor_column - ) - if warehouse_type is None: - return None - - declared: CursorType | None = _parse_cursor_type(cursor_type) - if declared is None: - return None - - if sql_analysis_enabled: - return _check_with_polyglot( - model_name=model_name, - cursor_column=cursor_column, - declared=declared, - warehouse_type=warehouse_type, - ) - - return _check_with_heuristic( - model_name=model_name, - cursor_column=cursor_column, - declared=declared, - warehouse_type=warehouse_type, - ) - - -def _find_column_type( - *, - columns: tuple[ColumnInfo, ...], - column_name: str, -) -> str | None: - """Find the warehouse type string for a column by name (case-insensitive).""" - - col: ColumnInfo - for col in columns: - if col.name.upper() == column_name.upper(): - return col.type - return None - - -def _parse_cursor_type(cursor_type: str) -> CursorType | None: - """Parse a cursor_type config string to the enum, or None if unrecognized.""" - - try: - return CursorType(cursor_type) - except ValueError: - return None - - -def _check_with_polyglot( - *, - model_name: str, - cursor_column: str, - declared: CursorType, - warehouse_type: str, -) -> PlanWarning | None: - """Classify warehouse type via sql_analysis and return error on clear mismatch.""" - - detected: CursorType | None = classify_cursor_sql_type(warehouse_type) - if detected is None: - return None - - if detected != declared: - return PlanWarning( - model_name=model_name, - severity=WarningSeverity.ERROR, - message=( - f"cursor column '{cursor_column}' has warehouse type '{warehouse_type}' " - f"which is {detected.value}, but cursor_type is '{declared.value}'" - ), - ) - - return None - - -def _check_with_heuristic( - *, - model_name: str, - cursor_column: str, - declared: CursorType, - warehouse_type: str, -) -> PlanWarning | None: - """Classify warehouse type via substring heuristic and warn on mismatch.""" - - detected: CursorType | None = classify_cursor_sql_type(warehouse_type) - if detected is None: - return None - - if detected != declared: - return PlanWarning( - model_name=model_name, - severity=WarningSeverity.WARNING, - message=( - f"cursor column '{cursor_column}' has warehouse type '{warehouse_type}' " - f"which appears to be {detected.value}, but cursor_type is '{declared.value}'" - ), - ) - - return None - - -def classify_cursor_sql_type(sql_type: str) -> CursorType | None: - """Classify a supported cross-adapter SQL cursor type.""" - - normalized: str = " ".join(sql_type.upper().split()) - decimal_match: re.Match[str] | None = _DECIMAL_INTEGER_TYPE_PATTERN.fullmatch(normalized) - if decimal_match is not None: - return CursorType.INTEGER if int(decimal_match.group(1) or 0) == 0 else None - - base_name: str = normalized.partition("(")[0].strip().replace(" ", "_") - if base_name in _POLYGLOT_TIMESTAMP_TYPE_NAMES: - return CursorType.TIMESTAMP - if base_name in _POLYGLOT_INTEGER_TYPE_NAMES: - return CursorType.INTEGER - - return _classify_type_with_polyglot(normalized) - - -def _classify_type_with_polyglot(warehouse_type: str) -> CursorType | None: - """Classify a warehouse type string using sql_analysis type parsing.""" - - polyglot_module: Any = import_polyglot() - - try: - parsed: Any = polyglot_module.parse_data_type(warehouse_type, dialect="generic") - except polyglot_module.PolyglotError as error: - log_debug_event( - logger=_DEBUG_LOGGER, - message="cursor type classification parse failed; falling back", - sqlbuild_warehouse_type=warehouse_type, - sqlbuild_error=str(error), - ) - return None - - args: dict[str, Any] = dict(getattr(parsed, "args", {}) or {}) - type_name: str = str(args.get("data_type", "")).upper() - if type_name == POLYGLOT_CUSTOM_DATA_TYPE_NAME: - type_name = str(args.get("name", "")).upper().replace("_", "") - - if type_name in _POLYGLOT_TIMESTAMP_TYPE_NAMES: - return CursorType.TIMESTAMP - if type_name in _POLYGLOT_INTEGER_TYPE_NAMES: - return CursorType.INTEGER - - return None diff --git a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py b/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py deleted file mode 100644 index 0d162c1326..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Planner-private microbatch range capping and capped-watermark validation.""" - -from __future__ import annotations - -from datetime import datetime - -from sqlbuild._native import aligned_timestamp_bounds, get_config_str, resolve_cursor_input_roles -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompileSqlReference, -) -from sqlbuild.compiler.compile.types import CompiledFunctionView, CompiledModelView -from sqlbuild.compiler.planner._helpers.resolve.lineage import resolve_lineage_reference -from sqlbuild.compiler.planner.main.execution.microbatch_limit import ( - _resolve_microbatch_limit_config, -) -from sqlbuild.compiler.planner.models import CursorBounds, Duration -from sqlbuild.compiler.planner.types import CursorType, IncrementalMode -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue -from sqlbuild.spec.contracts.types import MicrobatchLimitAction - -_CAP_ACTIONS: frozenset[MicrobatchLimitAction] = frozenset( - {MicrobatchLimitAction.CAP_FROM_START, MicrobatchLimitAction.CAP_FROM_END} -) - - -def cap_microbatch_bounds( - *, - bounds: CursorBounds, - batch_size: str, - cursor_type: str, - max_batches: int, - action: MicrobatchLimitAction, -) -> CursorBounds: - """Cap one concrete range to its earliest or latest batch windows.""" - - if action not in { - MicrobatchLimitAction.CAP_FROM_END, - MicrobatchLimitAction.CAP_FROM_START, - }: - return bounds - if cursor_type == CursorType.INTEGER: - if not isinstance(bounds.start, IntegerValue) or not isinstance(bounds.end, IntegerValue): - return bounds - start: int = bounds.start.value - end: int = bounds.end.value - try: - size: int = int(batch_size) - except ValueError: - return bounds - if size <= 0: - return bounds - if action == MicrobatchLimitAction.CAP_FROM_START: - return CursorBounds( - start=bounds.start, - end=IntegerValue(value=min(end, start + size * max_batches)), - ) - batch_count: int = max(0, (end - start + size - 1) // size) - skipped_batches: int = max(0, batch_count - max_batches) - return CursorBounds( - start=IntegerValue(value=start + size * skipped_batches), end=bounds.end - ) - - if cursor_type != CursorType.TIMESTAMP: - return bounds - duration: Duration | None = Duration.parse(batch_size) - if duration is None: - return bounds - if not isinstance(bounds.start, DateValue | TimestampValue) or not isinstance( - bounds.end, DateValue | TimestampValue - ): - return bounds - start_at, end_at = aligned_timestamp_bounds(start=bounds.start, end=bounds.end) - if action == MicrobatchLimitAction.CAP_FROM_START: - capped_end: datetime = start_at - for _ in range(max_batches): - capped_end = min(duration.add_to(capped_end), end_at) - return CursorBounds(start=bounds.start, end=TimestampValue(value=capped_end)) - boundaries: list[datetime] = [start_at] - while boundaries[-1] < end_at: - boundaries.append(min(duration.add_to(boundaries[-1]), end_at)) - capped_start: datetime = boundaries[max(0, len(boundaries) - 1 - max_batches)] - return CursorBounds(start=TimestampValue(value=capped_start), end=bounds.end) - - -def validate_capped_watermark_inputs( - *, - model: CompiledModelView, - models_by_name: dict[str, CompiledModelView], - functions_by_name: dict[str, CompiledFunctionView] | None = None, -) -> None: - """Reject capped producers feeding microbatch watermark inputs.""" - - if ( - get_config_str(values=model.config.values, key="incremental_mode") - != IncrementalMode.MICROBATCH - ): - return - input_name: str - for input_name in resolve_cursor_input_roles(model=model).watermark_inputs: - ref: CompileSqlReference = resolve_lineage_reference( - model=model, - input_name=input_name, - models_by_name=models_by_name, - functions_by_name=functions_by_name or {}, - ) - producer: CompiledModelView | None = _find_capped_producer_ancestor( - reference=ref, - models_by_name=models_by_name, - functions_by_name=functions_by_name or {}, - ) - if producer is None: - continue - if producer.name != input_name: - raise CompileInputError( - f"model '{model.name}' uses watermark input '{input_name}' derived from capped " - f"producer '{producer.name}'; capped producers cannot serve as watermark inputs" - ) - raise CompileInputError( - f"model '{model.name}' uses capped producer '{producer.name}' as a watermark input; " - "capped producers cannot serve as watermark inputs" - ) - - -def _find_capped_producer_ancestor( - *, - reference: CompileSqlReference, - models_by_name: dict[str, CompiledModelView], - functions_by_name: dict[str, CompiledFunctionView], -) -> CompiledModelView | None: - """Return the first capped microbatch model in a reference's upstream lineage.""" - - pending: list[CompileSqlReference] = [reference] - visited: set[tuple[str, str]] = set() - while pending: - current: CompileSqlReference = pending.pop(0) - identity: tuple[str, str] = (current.ref_kind, current.ref_name) - if identity in visited: - continue - visited.add(identity) - if current.ref_kind == SqlReferenceKind.REF: - producer: CompiledModelView | None = models_by_name.get(current.ref_name) - if producer is None: - continue - action: MicrobatchLimitAction | None = _resolve_microbatch_limit_config( - values=producer.config.values - )[1] - is_microbatch: bool = ( - get_config_str(values=producer.config.values, key="incremental_mode") - == IncrementalMode.MICROBATCH - ) - if is_microbatch and action in _CAP_ACTIONS: - return producer - if is_microbatch: - watermark_name: str - for watermark_name in resolve_cursor_input_roles(model=producer).watermark_inputs: - pending.append( - resolve_lineage_reference( - model=producer, - input_name=watermark_name, - models_by_name=models_by_name, - functions_by_name=functions_by_name, - ) - ) - else: - pending.extend(producer.references) - elif current.ref_kind in { - SqlReferenceKind.UDF, - SqlReferenceKind.TABLE_FUNCTION, - }: - function: CompiledFunctionView | None = functions_by_name.get(current.ref_name) - if function is not None: - pending.extend(function.references) - return None diff --git a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_count.py b/src/sqlbuild/compiler/planner/_helpers/output/microbatch_count.py deleted file mode 100644 index 247c33d9a8..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_count.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Planner-private microbatch count calculation.""" - -from __future__ import annotations - -from datetime import datetime - -from sqlbuild._native import aligned_timestamp_bounds -from sqlbuild.compiler.planner.models import CursorBounds, Duration -from sqlbuild.compiler.planner.types import CursorType -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue - - -def count_microbatches( - *, bounds: CursorBounds, batch_size: str, cursor_type: str, equal_bounds_are_batch: bool = False -) -> int: - """Count batches for one concrete cursor range.""" - - if cursor_type == CursorType.INTEGER: - if not isinstance(bounds.start, IntegerValue) or not isinstance(bounds.end, IntegerValue): - return 0 - start: int = bounds.start.value - end: int = bounds.end.value - try: - size: int = int(batch_size) - except ValueError: - return 0 - if equal_bounds_are_batch and start == end: - return 1 - if size <= 0 or start >= end: - return 0 - return (end - start + size - 1) // size - if cursor_type != CursorType.TIMESTAMP: - return 0 - duration: Duration | None = Duration.parse(batch_size) - if duration is None: - return 0 - if not isinstance(bounds.start, DateValue | TimestampValue) or not isinstance( - bounds.end, DateValue | TimestampValue - ): - return 0 - current: datetime - end_at: datetime - current, end_at = aligned_timestamp_bounds(start=bounds.start, end=bounds.end) - if equal_bounds_are_batch and current == end_at: - return 1 - count: int = 0 - while current < end_at: - current = min(duration.add_to(current), end_at) - count += 1 - return count diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index eec6e54e65..7a9d8f542e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -7,26 +7,39 @@ from decimal import Decimal from typing import Any -from sqlbuild._native import cursor_parse as parse -from sqlbuild._native import cursor_render as render -from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild._native import ( + build_model_warnings, + cap_microbatch_bounds, + check_cursor_type_consistency, + count_microbatches, + detect_model_changes, get_config_bool, get_config_cursor_bound, get_config_int, get_config_str, get_config_string_tuple, + get_materialization_type, microbatch_guidance, + microbatch_limit_warning, render_source_relation, + replaces_incremental_relation, resolve_bounded_cursor_override, resolve_cursor_input_roles, resolve_effective_microbatch_batch_size, + resolve_lineage_reference, + resolve_microbatch_limit_config, + resolve_model_plan_action, + resolve_schema_actions, + source_binding_diagnostics, + validate_capped_watermark_inputs, ) +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._source_bindings import get_source_binding_diagnostics from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, @@ -45,20 +58,6 @@ ) from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.graph.source_load_nodes import build_source_load_map -from sqlbuild.compiler.planner._helpers.output.cursor_type_check import ( - check_cursor_type_consistency, -) -from sqlbuild.compiler.planner._helpers.output.microbatch_bounds import ( - cap_microbatch_bounds, - validate_capped_watermark_inputs, -) -from sqlbuild.compiler.planner._helpers.output.microbatch_count import count_microbatches -from sqlbuild.compiler.planner._helpers.output.strategy import ( - build_model_warnings, - get_materialization_type, - resolve_model_plan_action, - resolve_schema_actions, -) from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( check_full_rebuild_allowed, describe_full_rebuild_cause, @@ -75,7 +74,6 @@ resolve_future_cursor_config, resolve_start_cursor_config, ) -from sqlbuild.compiler.planner._helpers.resolve.lineage import resolve_lineage_reference from sqlbuild.compiler.planner._helpers.resolve.refs import ( apply_deferred_locations, build_function_locations, @@ -96,17 +94,9 @@ ) from sqlbuild.compiler.planner.constants import MICROBATCH_CONCURRENCY_MESSAGE from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.changes._model_changes import detect_model_changes -from sqlbuild.compiler.planner.main.changes.relation_replacement import ( - replaces_incremental_relation, -) from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety from sqlbuild.compiler.planner.main.execution.future_cursor_warning import future_cursor_cap_warning from sqlbuild.compiler.planner.main.execution.maximum_start_warning import maximum_start_cap_warning -from sqlbuild.compiler.planner.main.execution.microbatch_limit import ( - _resolve_microbatch_limit_config, - microbatch_limit_warning, -) from sqlbuild.compiler.planner.models import ( BackfillResult, ChangeDetectionResult, @@ -388,7 +378,7 @@ def build_plan_entries( for diagnostic in project.diagnostics if diagnostic.is_error and diagnostic.resource_type != CompiledResourceType.SQL_TEST ), - *get_source_binding_diagnostics( + *source_binding_diagnostics( project=project, columns=relations.source_warehouse_columns, profile=adapter.expression_inference_profile(), @@ -930,8 +920,8 @@ def plan_model_from_change( values=model.config.values, key="unaccounted_partition_policy" ), microbatch_range=microbatch_range, - microbatch_limit=_resolve_microbatch_limit_config(values=model.config.values)[0], - declared_microbatch_limit_action=_resolve_microbatch_limit_config( + microbatch_limit=resolve_microbatch_limit_config(values=model.config.values)[0], + declared_microbatch_limit_action=resolve_microbatch_limit_config( values=model.config.values )[1], start_cursor_override=start_cursor_override, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py deleted file mode 100644 index 59f9967230..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py +++ /dev/null @@ -1,389 +0,0 @@ -"""Per-model plan action resolution, schema action resolution, and logical DDL generation.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.discovery.constants import ( - PROJECT_CONFIG_FILENAME, - QUERY_CHANGE_TRACKING_SETTING_KEY, - SETTINGS_SECTION, -) -from sqlbuild.compiler.planner.constants import RECORDED_RELATION_MISSING_WARNING_CODE -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - ChangeDetectionResult, - PlanWarning, - SchemaAction, - SchemaFinding, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - ChangeKind, - IncrementalStrategy, - MaterializationType, - OnSchemaChange, - PlanAction, - PlanReason, - SchemaActionKind, - SchemaChangeKind, - SchemaColumnSource, - WarningSeverity, -) -from sqlbuild.errors.setting_help.main.setting_help import setting_help - -_DEFAULT_ON_SCHEMA_CHANGE: OnSchemaChange = OnSchemaChange.APPEND_NEW_COLUMNS -_OWN_CHANGE_REASONS: dict[ChangeKind, PlanReason] = { - ChangeKind.FIRST_RUN: PlanReason.FIRST_RUN, - ChangeKind.QUERY_CHANGED: PlanReason.QUERY_CHANGED, - ChangeKind.FUNCTION_CHANGED: PlanReason.FUNCTION_CHANGED, - ChangeKind.CONFIG_CHANGED: PlanReason.CONFIG_CHANGED, - ChangeKind.SCHEMA_CHANGED: PlanReason.SCHEMA_CHANGED, -} -_VIEW_CHANGE_REASONS: dict[ChangeKind, PlanReason] = { - **_OWN_CHANGE_REASONS, - ChangeKind.RENAMED: PlanReason.RENAMED, -} - - -def resolve_model_plan_action( - *, - model: CompiledModelView, - change_result: ChangeDetectionResult, - full_refresh: bool, -) -> tuple[PlanAction, PlanReason]: - """Determine the plan action and reason for a single model.""" - - if _is_disabled(model): - return PlanAction.SKIP, PlanReason.DISABLED - - materialization: MaterializationType = get_materialization_type(model) - - if materialization == MaterializationType.CUSTOM: - return PlanAction.CUSTOM, _custom_reason( - change_result=change_result, full_refresh=full_refresh - ) - - if materialization == MaterializationType.VIEW: - return PlanAction.CREATE_VIEW, _view_reason( - change_result=change_result, full_refresh=full_refresh - ) - - if materialization == MaterializationType.SNAPSHOT: - return PlanAction.SNAPSHOT, _snapshot_reason( - change_result=change_result, full_refresh=full_refresh - ) - - if full_refresh: - return PlanAction.CREATE_TABLE, PlanReason.FULL_REFRESH - - if change_result.change_kind == ChangeKind.FIRST_RUN: - return PlanAction.CREATE_TABLE, PlanReason.FIRST_RUN - - if change_result.change_kind == ChangeKind.RENAMED: - return PlanAction.CREATE_TABLE, PlanReason.RENAMED - - if change_result.backfill.action == BackfillAction.FULL: - reason: PlanReason = _backfill_reason(change_result) - return PlanAction.CREATE_TABLE, reason - - if materialization == MaterializationType.TABLE: - return _table_action(change_result) - - return _incremental_action(model=model, change_result=change_result) - - -def resolve_schema_actions( - *, - schema_findings: tuple[SchemaFinding, ...], - on_schema_change: OnSchemaChange | None, -) -> tuple[SchemaAction, ...]: - """Resolve concrete schema change actions from findings and on_schema_change config.""" - - effective: OnSchemaChange = on_schema_change or _DEFAULT_ON_SCHEMA_CHANGE - - if effective == OnSchemaChange.IGNORE or effective == OnSchemaChange.FAIL: - return () - - actions: list[SchemaAction] = [] - finding: SchemaFinding - for finding in schema_findings: - if effective == OnSchemaChange.APPEND_NEW_COLUMNS: - if finding.kind == SchemaChangeKind.COLUMN_ADDED: - actions.append( - SchemaAction( - kind=SchemaActionKind.ADD_COLUMN, - column_name=finding.column_name, - column_type=finding.expected_type, - ) - ) - elif effective == OnSchemaChange.SYNC_ALL_COLUMNS: - if finding.kind == SchemaChangeKind.COLUMN_ADDED: - actions.append( - SchemaAction( - kind=SchemaActionKind.ADD_COLUMN, - column_name=finding.column_name, - column_type=finding.expected_type, - ) - ) - elif finding.kind == SchemaChangeKind.COLUMN_REMOVED: - actions.append( - SchemaAction( - kind=SchemaActionKind.DROP_COLUMN, - column_name=finding.column_name, - ) - ) - elif finding.kind == SchemaChangeKind.COLUMN_TYPE_CHANGED: - actions.append( - SchemaAction( - kind=SchemaActionKind.ALTER_COLUMN_TYPE, - column_name=finding.column_name, - column_type=finding.expected_type, - ) - ) - - return tuple(actions) - - -def build_model_warnings( - *, - model_name: str, - materialization_type: MaterializationType, - change_result: ChangeDetectionResult, - schema_actions: tuple[SchemaAction, ...], - on_schema_change: OnSchemaChange | None, - type_enforcement: bool, -) -> tuple[PlanWarning, ...]: - """Build warnings for a single model based on change detection and plan decisions.""" - - effective: OnSchemaChange = on_schema_change or _DEFAULT_ON_SCHEMA_CHANGE - warnings: list[PlanWarning] = [] - - if change_result.recorded_build_relation_missing: - warnings.append( - PlanWarning( - model_name=model_name, - severity=WarningSeverity.WARNING, - code=RECORDED_RELATION_MISSING_WARNING_CODE, - message=( - f"{model_name}: SQLBuild state records a previous build, but the relation " - "no longer exists in the warehouse. It may have been dropped outside " - "SQLBuild; planning a first run to recreate it." - ), - ) - ) - - if effective == OnSchemaChange.FAIL and change_result.schema_findings: - warnings.append( - PlanWarning( - model_name=model_name, - severity=WarningSeverity.ERROR, - message="schema change detected and on_schema_change is set to fail", - ) - ) - - finding: SchemaFinding - for finding in change_result.schema_findings: - if type_enforcement and finding.source == SchemaColumnSource.YML: - if finding.kind == SchemaChangeKind.COLUMN_TYPE_CHANGED: - warnings.append( - PlanWarning( - model_name=model_name, - severity=WarningSeverity.WARNING, - message=( - f"enforced column {finding.column_name} type mismatch: " - f"expected {finding.expected_type}, " - f"warehouse has {finding.actual_type}" - ), - ) - ) - elif finding.source == SchemaColumnSource.SQL_ANALYSIS: - if finding.kind in ( - SchemaChangeKind.COLUMN_ADDED, - SchemaChangeKind.COLUMN_TYPE_CHANGED, - ): - warnings.append( - PlanWarning( - model_name=model_name, - severity=WarningSeverity.INFO, - message=( - f"inferred column {finding.column_name}: {_describe_finding(finding)}" - ), - ) - ) - - if ( - materialization_type == MaterializationType.INCREMENTAL - and (change_result.query_changed or change_result.changed_functions) - and change_result.backfill.action == BackfillAction.FORWARD_ONLY - ): - changed_subject: str = ( - "the model's SQL changed" - if change_result.query_changed - else f"function {', '.join(change_result.changed_functions)} changed" - ) - warnings.append( - PlanWarning( - model_name=model_name, - severity=WarningSeverity.WARNING, - message=( - f"{model_name}: {changed_subject}, so already-built dates keep " - "their old results - only new dates will use the new query. To apply " - "the change to existing data, choose how much to reprocess: " - "replay_on_change full (rebuild every date with the new SQL), " - "replay_on_change bounded- e.g. bounded-14d (reprocess just " - "that window), or leave it forward-only (existing dates as-is, new " - "dates use the new query - the default). If this model's output does " - "not depend on the SQL that changed, this is safe to ignore. " - + setting_help( - purpose="To silence these warnings", - file_name=PROJECT_CONFIG_FILENAME, - section=SETTINGS_SECTION, - key=QUERY_CHANGE_TRACKING_SETTING_KEY, - value=False, - ) - ), - ) - ) - - has_schema_backfill_findings: bool = any( - f.kind in (SchemaChangeKind.COLUMN_ADDED, SchemaChangeKind.COLUMN_TYPE_CHANGED) - for f in change_result.schema_findings - ) - if has_schema_backfill_findings and not schema_actions: - if effective == OnSchemaChange.IGNORE: - warnings.append( - PlanWarning( - model_name=model_name, - severity=WarningSeverity.INFO, - message="schema change detected but on_schema_change is set to ignore", - ) - ) - - return tuple(warnings) - - -def _is_disabled(model: CompiledModelView) -> bool: - """Check whether a model is explicitly disabled.""" - - raw: object | None = model.config.values.get("enabled") - if isinstance(raw, bool): - return not raw - return False - - -def _custom_reason(*, change_result: ChangeDetectionResult, full_refresh: bool) -> PlanReason: - """Determine the reason for a custom materialization action.""" - - if full_refresh: - return PlanReason.FULL_REFRESH - return _OWN_CHANGE_REASONS.get(change_result.change_kind, PlanReason.NO_CHANGE) - - -def _view_reason(*, change_result: ChangeDetectionResult, full_refresh: bool) -> PlanReason: - """Determine the reason for a view action.""" - - if full_refresh: - return PlanReason.FULL_REFRESH - return _VIEW_CHANGE_REASONS.get(change_result.change_kind, PlanReason.NO_CHANGE) - - -def _snapshot_reason(*, change_result: ChangeDetectionResult, full_refresh: bool) -> PlanReason: - """Determine the reason for a snapshot materialization action.""" - - if full_refresh: - return PlanReason.FULL_REFRESH - return _OWN_CHANGE_REASONS.get(change_result.change_kind, PlanReason.NORMAL_INCREMENTAL) - - -def _backfill_reason(change_result: ChangeDetectionResult) -> PlanReason: - """Determine the reason when backfill forces a full rebuild.""" - - if change_result.query_changed: - return PlanReason.QUERY_CHANGED - if change_result.changed_functions: - return PlanReason.FUNCTION_CHANGED - if change_result.config_changed: - return PlanReason.CONFIG_CHANGED - if change_result.schema_findings: - return PlanReason.SCHEMA_CHANGED - return PlanReason.FULL_REFRESH - - -def _table_action( - change_result: ChangeDetectionResult, -) -> tuple[PlanAction, PlanReason]: - """Determine action for a table materialization (non-incremental).""" - - if change_result.change_kind == ChangeKind.QUERY_CHANGED: - return PlanAction.CREATE_TABLE, PlanReason.QUERY_CHANGED - if change_result.change_kind == ChangeKind.FUNCTION_CHANGED: - return PlanAction.CREATE_TABLE, PlanReason.FUNCTION_CHANGED - if change_result.change_kind == ChangeKind.CONFIG_CHANGED: - return PlanAction.CREATE_TABLE, PlanReason.CONFIG_CHANGED - if change_result.change_kind == ChangeKind.SCHEMA_CHANGED: - return PlanAction.CREATE_TABLE, PlanReason.SCHEMA_CHANGED - return PlanAction.CREATE_TABLE, PlanReason.NO_CHANGE - - -def _incremental_action( - *, - model: CompiledModelView, - change_result: ChangeDetectionResult, -) -> tuple[PlanAction, PlanReason]: - """Determine action for an incremental materialization.""" - - raw_strategy: object | None = model.config.values.get("incremental_strategy") - if not isinstance(raw_strategy, str): - raise PlannerInputError( - f"incremental model '{model.name}' is missing required incremental_strategy", - code="S201", - ) - - reason: PlanReason - if change_result.change_kind == ChangeKind.QUERY_CHANGED: - reason = PlanReason.QUERY_CHANGED - elif change_result.change_kind == ChangeKind.FUNCTION_CHANGED: - reason = PlanReason.FUNCTION_CHANGED - elif change_result.change_kind == ChangeKind.CONFIG_CHANGED: - reason = PlanReason.CONFIG_CHANGED - elif change_result.change_kind == ChangeKind.SCHEMA_CHANGED: - reason = PlanReason.SCHEMA_CHANGED - else: - reason = PlanReason.NORMAL_INCREMENTAL - - action_map: dict[str, PlanAction] = { - IncrementalStrategy.APPEND: PlanAction.INCREMENTAL_APPEND, - IncrementalStrategy.DELETE_INSERT: PlanAction.INCREMENTAL_DELETE_INSERT, - IncrementalStrategy.MERGE: PlanAction.INCREMENTAL_MERGE, - } - action: PlanAction | None = action_map.get(raw_strategy) - if action is None: - raise PlannerInputError( - f"incremental model '{model.name}' has unknown strategy '{raw_strategy}'", - code="S202", - ) - - return action, reason - - -def _describe_finding(finding: SchemaFinding) -> str: - """Build a human-readable description for one schema finding.""" - - if finding.kind == SchemaChangeKind.COLUMN_ADDED: - type_suffix: str = f" ({finding.expected_type})" if finding.expected_type else "" - return f"new column not in warehouse{type_suffix}" - if finding.kind == SchemaChangeKind.COLUMN_TYPE_CHANGED: - return f"expected {finding.expected_type}, warehouse has {finding.actual_type}" - return f"{finding.kind.value}" - - -def get_materialization_type(model: CompiledModelView) -> MaterializationType: - """Extract materialization type from model config.""" - - raw: object | None = model.config.values.get("materialized") - if isinstance(raw, str): - try: - return MaterializationType(raw) - except ValueError: - return MaterializationType.CUSTOM - return MaterializationType.TABLE diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/identities.py b/src/sqlbuild/compiler/planner/_helpers/planning/identities.py index a7e2a38b40..1be7c7f043 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/identities.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/identities.py @@ -4,9 +4,9 @@ from dataclasses import replace +from sqlbuild._native import detect_model_changes_in_scope from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.planner._helpers.changes.detect import detect_model_changes_in_scope from sqlbuild.compiler.planner._helpers.identity.direct import ( build_direct_model_version_identities, ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/retention.py b/src/sqlbuild/compiler/planner/_helpers/planning/retention.py index 8490daf239..2e508119f6 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/retention.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/retention.py @@ -6,6 +6,7 @@ from collections import defaultdict from dataclasses import replace +from sqlbuild._native import resolve_target_config from sqlbuild.adapter.contract.models import ( RelationInfo, RelationLookup, @@ -28,7 +29,6 @@ RetentionDirection, RetentionPlanPhase, ) -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention, TargetConfig from sqlbuild.spec.contracts.types import TableType diff --git a/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py b/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py index 2853d9a560..74d5c886b7 100644 --- a/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py +++ b/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import detect_model_changes from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.fingerprints.models import Fingerprint @@ -9,7 +10,6 @@ classify_selection_staleness_warnings, format_stale_upstream_warning_message, ) -from sqlbuild.compiler.planner.main.changes._model_changes import detect_model_changes from sqlbuild.compiler.planner.models import ( ChangeDetectionResult, DirectModelVersionIdentities, diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py index 7bbffc5884..b819708b61 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py @@ -3,8 +3,8 @@ from __future__ import annotations from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token +from sqlbuild._native import resolve_cursor_intrinsics from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics from sqlbuild.compiler.planner.models import CursorBounds diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/lineage.py b/src/sqlbuild/compiler/planner/_helpers/resolve/lineage.py deleted file mode 100644 index 1e3929e4d5..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/lineage.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Resolve declared cursor inputs from transitive SQL resource lineage.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import ( - CompileSqlReference, -) -from sqlbuild.compiler.compile.types import CompiledFunctionView, CompiledModelView -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.references.types import SqlReferenceKind - - -def resolve_lineage_reference( - *, - model: CompiledModelView, - input_name: str, - models_by_name: dict[str, CompiledModelView], - functions_by_name: dict[str, CompiledFunctionView], -) -> CompileSqlReference: - """Resolve one named input through model and SQL function references.""" - - pending: list[CompileSqlReference] = list(model.references) - visited: set[tuple[str, str]] = set() - while pending: - reference: CompileSqlReference = pending.pop(0) - if reference.ref_name == input_name: - return reference - identity: tuple[str, str] = (reference.ref_kind, reference.ref_name) - if identity in visited: - continue - visited.add(identity) - if reference.ref_kind == SqlReferenceKind.REF: - upstream_model: CompiledModelView | None = models_by_name.get(reference.ref_name) - if upstream_model is not None: - pending.extend(upstream_model.references) - elif reference.ref_kind in { - SqlReferenceKind.UDF, - SqlReferenceKind.TABLE_FUNCTION, - }: - upstream_function: CompiledFunctionView | None = functions_by_name.get( - reference.ref_name - ) - if upstream_function is not None: - pending.extend(upstream_function.references) - raise PlannerInputError( - f"model '{model.name}': cursor_inputs watermark references '{input_name}', but it is " - "not in the model's upstream lineage", - code="S302", - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py index 1f7a3c182c..ea4cf8a5be 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py @@ -27,26 +27,25 @@ SQL_IDENTIFIER_LEADING_CHARACTERS, ) from sqlbuild.compiler.planner.models import CursorBounds -from sqlbuild.compiler.references.main._quoted_reference_call_pattern import ( - quoted_reference_call_pattern, +from sqlbuild.compiler.references.constants import ( + DBT_REF_CALL_PREFIX, + REF_PATTERN, + SEED_PATTERN, + TABLE_FUNCTION_PATTERN, + UDF_PATTERN, ) -from sqlbuild.compiler.references.main._reference_call_prefix_pattern_text import ( - reference_call_prefix_pattern_text, -) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind +from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -_REF_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.REF) -_SEED_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.SEED) +_REF_PATTERN: re.Pattern[str] = REF_PATTERN +_SEED_PATTERN: re.Pattern[str] = SEED_PATTERN _DBT_REF_PATTERN: re.Pattern[str] = re.compile( - rf'{reference_call_prefix_pattern_text(SqlReferenceKind.DBT_REF)}\s*"([^"]+)"\s*' + rf'{DBT_REF_CALL_PREFIX}\s*"([^"]+)"\s*' r'(?:,\s*"([^"]+)"\s*)?\)', re.IGNORECASE, ) -_UDF_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.UDF) -_TABLE_FUNCTION_PATTERN: re.Pattern[str] = quoted_reference_call_pattern( - SqlReferenceKind.TABLE_FUNCTION -) +_UDF_PATTERN: re.Pattern[str] = UDF_PATTERN +_TABLE_FUNCTION_PATTERN: re.Pattern[str] = TABLE_FUNCTION_PATTERN _REFERENCE_SCAN_CONTEXT: str = "SQL reference resolution" _CLAUSE_KEYWORDS: frozenset[str] = frozenset( { diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py index eee9f5e4be..2a1d38d872 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py @@ -11,10 +11,10 @@ get_config_str, resolve_bounded_cursor_override, resolve_cursor_input_roles, + resolve_cursor_intrinsics, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationLookup -from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics from sqlbuild.compiler.compile.models import ( CompiledRelationLocation, CursorInputRoles, diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py b/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py index 1ab40f1157..729848a6f7 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py @@ -18,13 +18,10 @@ from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import CursorBounds from sqlbuild.compiler.planner.types import ContractPolicy -from sqlbuild.compiler.references.main._quoted_reference_call_pattern import ( - quoted_reference_call_pattern, -) -from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.references.constants import SOURCE_PATTERN from sqlbuild.spec.contracts.models import SourceColumnEntry, SourceEntry -_SOURCE_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.SOURCE) +_SOURCE_PATTERN: re.Pattern[str] = SOURCE_PATTERN _DERIVED_TABLE_ALIAS_KEYWORDS: frozenset[str] = frozenset( { "APPLY", diff --git a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py index 02625c3ac9..17d3d343e8 100644 --- a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py +++ b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import get_materialization_type from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -24,7 +25,6 @@ from sqlbuild.compiler.planner._helpers.identity.functions import ( build_compiled_function_fingerprint_sql, ) -from sqlbuild.compiler.planner._helpers.output.strategy import get_materialization_type from sqlbuild.compiler.planner._helpers.resolve.refs import ( build_function_locations, build_model_locations, diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py index b4aed7b733..385e1dcffb 100644 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py +++ b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py @@ -79,26 +79,21 @@ FixtureKey, ScenarioArtifactKind, ) -from sqlbuild.compiler.references.main._reference_call_prefix_pattern_text import ( - reference_call_prefix_pattern_text, +from sqlbuild.compiler.references.constants import ( + DBT_REF_CALL_PREFIX, + REF_CALL_PREFIX, + SEED_CALL_PREFIX, + SOURCE_CALL_PREFIX, ) -from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import SourceEntry _NAME_ARGUMENT_PATTERN_TEXT: str = r'"(?P[^"]+)"\)' -_REF_PATTERN: re.Pattern[str] = re.compile( - reference_call_prefix_pattern_text(SqlReferenceKind.REF) + _NAME_ARGUMENT_PATTERN_TEXT -) -_SEED_PATTERN: re.Pattern[str] = re.compile( - reference_call_prefix_pattern_text(SqlReferenceKind.SEED) + _NAME_ARGUMENT_PATTERN_TEXT -) -_SOURCE_PATTERN: re.Pattern[str] = re.compile( - reference_call_prefix_pattern_text(SqlReferenceKind.SOURCE) + _NAME_ARGUMENT_PATTERN_TEXT -) +_REF_PATTERN: re.Pattern[str] = re.compile(REF_CALL_PREFIX + _NAME_ARGUMENT_PATTERN_TEXT) +_SEED_PATTERN: re.Pattern[str] = re.compile(SEED_CALL_PREFIX + _NAME_ARGUMENT_PATTERN_TEXT) +_SOURCE_PATTERN: re.Pattern[str] = re.compile(SOURCE_CALL_PREFIX + _NAME_ARGUMENT_PATTERN_TEXT) _DBT_REF_PATTERN: re.Pattern[str] = re.compile( - reference_call_prefix_pattern_text(SqlReferenceKind.DBT_REF) - + r'"(?P[^"]+)"(?:\s*,\s*"(?P[^"]+)")?\)' + DBT_REF_CALL_PREFIX + r'"(?P[^"]+)"(?:\s*,\s*"(?P[^"]+)")?\)' ) diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py index 3df6a798ef..31bf70ada0 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py @@ -8,10 +8,9 @@ from sqlbuild._native import cursor_floor_to_grain as floor_to_grain from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render -from sqlbuild._native import get_config_str +from sqlbuild._native import get_config_str, resolve_cursor_intrinsics from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics from sqlbuild.compiler.compile.models import CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner._helpers.resolve.cursor_intrinsics import ( diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py index 41dd53f435..ccff286819 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py @@ -9,20 +9,22 @@ from datetime import datetime from typing import Any -from sqlbuild._native import cursor_compare as compare -from sqlbuild._native import cursor_exclusive_to_inclusive as exclusive_to_inclusive -from sqlbuild._native import cursor_observed_partition as observed_partition -from sqlbuild._native import cursor_parse as parse -from sqlbuild._native import cursor_render as render from sqlbuild._native import ( + classify_cursor_sql_type, get_config_cursor_bound, get_config_str, read_latest_fingerprints, render_source_relation, resolve_cursor_input_roles, + resolve_lineage_reference, select_latest_fingerprints, transitive_closure_many, ) +from sqlbuild._native import cursor_compare as compare +from sqlbuild._native import cursor_exclusive_to_inclusive as exclusive_to_inclusive +from sqlbuild._native import cursor_observed_partition as observed_partition +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapter.contract.types import AdapterExecute @@ -66,7 +68,6 @@ from sqlbuild.compiler.planner._helpers.graph.loader_dag import ( build_upstream_intermediate_source_map, ) -from sqlbuild.compiler.planner._helpers.output.cursor_type_check import classify_cursor_sql_type from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( effectively_full_refreshed_model_names, ) @@ -75,7 +76,6 @@ resolve_effective_timestamp_grain, ) from sqlbuild.compiler.planner._helpers.resolve.cursor_policies import resolve_start_cursor_config -from sqlbuild.compiler.planner._helpers.resolve.lineage import resolve_lineage_reference from sqlbuild.compiler.planner._helpers.resolve.maximum_start import maximum_allowed_start from sqlbuild.compiler.planner.constants import METADATA_NAME_FILTER_LIMIT from sqlbuild.compiler.planner.exceptions import PlannerInputError diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py index 83d4c44c15..78a348b44a 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py @@ -5,7 +5,7 @@ import re from dataclasses import replace -from sqlbuild._native import render_project_var_text +from sqlbuild._native import render_project_var_text, resolve_target_config from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) @@ -23,17 +23,14 @@ SOURCE_DEFERRAL_CONTEXT_FIELDS, ) from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.references.main._quoted_reference_call_pattern import ( - quoted_reference_call_pattern, -) +from sqlbuild.compiler.references.constants import SOURCE_PATTERN from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceEntry, TargetConfig _CTX_PATTERN: re.Pattern[str] = re.compile(r"\$\{CTX:([^}]+)\}") _VAR_PATTERN: re.Pattern[str] = re.compile(r"\$\{([^}:]+)\}") -_SOURCE_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.SOURCE) +_SOURCE_PATTERN: re.Pattern[str] = SOURCE_PATTERN def build_source_read_map( diff --git a/src/sqlbuild/compiler/planner/classes/fixture_column_inferences.py b/src/sqlbuild/compiler/planner/classes/fixture_column_inferences.py index 0bd817703e..16025222bf 100644 --- a/src/sqlbuild/compiler/planner/classes/fixture_column_inferences.py +++ b/src/sqlbuild/compiler/planner/classes/fixture_column_inferences.py @@ -2,8 +2,8 @@ from __future__ import annotations +from sqlbuild._native import infer_fixture_column_facts from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.main._infer_fixture_columns import infer_fixture_column_facts from sqlbuild.compiler.compile.models import FixtureColumnInference diff --git a/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py b/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py index ddc30ab436..d31d5f9e7e 100644 --- a/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py +++ b/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any, cast -from sqlbuild.compiler.planner._helpers.changes.metadata import without_declared_columns_hash +from sqlbuild._native import without_declared_columns_hash from sqlbuild.compiler.planner._helpers.migrations.fingerprint import ( build_migration_fingerprint, migration_fingerprint_ref_names, diff --git a/src/sqlbuild/compiler/planner/main/changes/_model_changes.py b/src/sqlbuild/compiler/planner/main/changes/_model_changes.py deleted file mode 100644 index d6f30f069b..0000000000 --- a/src/sqlbuild/compiler/planner/main/changes/_model_changes.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Public entrypoint for one-model change detection.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner._helpers.changes.detect import ( - detect_model_changes as _detect_model_changes, -) -from sqlbuild.compiler.planner.models import ChangeDetectionResult, WarehouseSnapshot - - -def detect_model_changes( - *, - model: CompiledModelView, - snapshot: WarehouseSnapshot, - sql_analysis_enabled: bool, - query_change_tracking: bool, - full_refresh: bool, - function_local_hashes: dict[str, str] | None = None, - expected_version_hash: str | None = None, - expected_metadata_json: str | None = None, -) -> ChangeDetectionResult: - """Detect changes for one model and resolve backfill policy.""" - - return _detect_model_changes( - model=model, - snapshot=snapshot, - sql_analysis_enabled=sql_analysis_enabled, - query_change_tracking=query_change_tracking, - full_refresh=full_refresh, - function_local_hashes=function_local_hashes, - expected_version_hash=expected_version_hash, - expected_metadata_json=expected_metadata_json, - ) diff --git a/src/sqlbuild/compiler/planner/main/changes/relation_replacement.py b/src/sqlbuild/compiler/planner/main/changes/relation_replacement.py deleted file mode 100644 index 5d53c18497..0000000000 --- a/src/sqlbuild/compiler/planner/main/changes/relation_replacement.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public incremental relation-replacement predicate.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction - - -def replaces_incremental_relation( - *, materialization_type: MaterializationType, action: PlanAction -) -> bool: - """Identify incremental replacements whose bounds must ignore the destination maximum.""" - - return ( - materialization_type == MaterializationType.INCREMENTAL - and action == PlanAction.CREATE_TABLE - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/display_plan.py b/src/sqlbuild/compiler/planner/main/execution/display_plan.py index b962f8575d..549d9a0158 100644 --- a/src/sqlbuild/compiler/planner/main/execution/display_plan.py +++ b/src/sqlbuild/compiler/planner/main/execution/display_plan.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild._native import get_config_str +from sqlbuild._native import get_config_str, get_materialization_type from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) @@ -11,7 +11,6 @@ build_downstream_deps, build_execution_upstream_deps, ) -from sqlbuild.compiler.planner._helpers.output.strategy import get_materialization_type from sqlbuild.compiler.planner._helpers.planning.full_refresh import resolve_model_full_refresh from sqlbuild.compiler.planner.models import BackfillResult, ModelPlanEntry, PlanOutput from sqlbuild.compiler.planner.types import ( diff --git a/src/sqlbuild/compiler/planner/main/execution/execution.py b/src/sqlbuild/compiler/planner/main/execution/execution.py index 12a33f9d6a..d2977a7cb6 100644 --- a/src/sqlbuild/compiler/planner/main/execution/execution.py +++ b/src/sqlbuild/compiler/planner/main/execution/execution.py @@ -7,6 +7,7 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import detect_changes, resolve_model_actions from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.compiler.compile.types import ( @@ -14,8 +15,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner._helpers.changes.actions import resolve_model_actions -from sqlbuild.compiler.planner._helpers.changes.detect import detect_changes from sqlbuild.compiler.planner._helpers.planning.buildability import ( check_selected_scope_buildability, ) diff --git a/src/sqlbuild/compiler/planner/main/execution/microbatch_limit.py b/src/sqlbuild/compiler/planner/main/execution/microbatch_limit.py deleted file mode 100644 index 2d7677f5c7..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/microbatch_limit.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Microbatch planning limit decisions shared by planner and runtime.""" - -from collections.abc import Mapping -from typing import cast - -from sqlbuild.spec.contracts.types import MicrobatchLimitAction - - -def _resolve_microbatch_limit_config( - *, values: Mapping[str, object] -) -> tuple[int | None, MicrobatchLimitAction | None]: - """Return the configured limit and explicit action from validated model values.""" - - raw: object | None = values.get("microbatch_limit") - if isinstance(raw, dict): - raw_config: Mapping[str, object] = cast("Mapping[str, object]", raw) - max_batches: object | None = raw_config.get("max_batches") - action: object | None = raw_config.get("action") - if ( - isinstance(max_batches, int) - and not isinstance(max_batches, bool) - and max_batches > 0 - and isinstance(action, str) - ): - return max_batches, MicrobatchLimitAction(action) - legacy_limit: object | None = values.get("max_microbatches") - return ( - legacy_limit - if isinstance(legacy_limit, int) and not isinstance(legacy_limit, bool) and legacy_limit > 0 - else None, - None, - ) - - -def microbatch_limit_warning( - *, model_name: str, max_batches: int | None, batch_count: int, action: MicrobatchLimitAction -) -> str | None: - """Return the prominent warning/error message when one model exceeds its limit.""" - - if max_batches is None or batch_count <= max_batches: - return None - return ( - f"MICROBATCH LIMIT EXCEEDED: model '{model_name}' planned {batch_count} batches, " - f"above the per-model limit of {max_batches} (action={action.value}). " - + ( - f"Selected {max_batches} batches and deferred {batch_count - max_batches}." - if action - in { - MicrobatchLimitAction.CAP_FROM_END, - MicrobatchLimitAction.CAP_FROM_START, - } - else ( - "Use --max-microbatches with an intentional invocation-specific value for " - "this backfill." - ) - ) - ) diff --git a/src/sqlbuild/compiler/project_assembly/_helpers/__init__.py b/src/sqlbuild/compiler/project_assembly/_helpers/__init__.py deleted file mode 100644 index 2a1bd2a676..0000000000 --- a/src/sqlbuild/compiler/project_assembly/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native project assembly helpers.""" diff --git a/src/sqlbuild/compiler/project_assembly/_helpers/request.py b/src/sqlbuild/compiler/project_assembly/_helpers/request.py deleted file mode 100644 index 2c890dfe2f..0000000000 --- a/src/sqlbuild/compiler/project_assembly/_helpers/request.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Build the native project assembly request from Python's compile inputs.""" - -from __future__ import annotations - -import os -from collections.abc import Iterable - -import sqlbuild._native as _native -from sqlbuild._native import render_project_var_text -from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.project_assembly.constants import ( - BOOLEAN_VARIABLE, - FALSE_TEXT, - NATIVE_INT_MAX, - NATIVE_INT_MIN, - NONE_VARIABLE, - OPAQUE_VARIABLE, - RENDER_LABEL, - RENDER_RAISES, - RENDER_REJECTED, - RENDERED_TEXT, - TEXT_VARIABLE, - TRUE_TEXT, -) -from sqlbuild.compiler.project_assembly.types import ( - ProjectRequestRow, - ReferenceRow, - SyntaxCheckRow, - VariableRow, -) -from sqlbuild.spec.contracts.models import DefaultsConfig, TargetConfig - - -def project_request( - *, - inputs: CompileInputsView, - dialect: str | None, - syntax_checks: tuple[tuple[tuple[str, dict[str, str] | None], ...], ...], -) -> tuple[ProjectRequestRow, list[Exception | None]]: - """Return the request row and, per variable, the exception rendering it as text raises.""" - - renders: list[tuple[str, str] | Exception | None] = [ - _opaque_render(value=value) for value in inputs.effective_vars.values() - ] - target: TargetConfig | None = inputs.effective_target - defaults: DefaultsConfig = inputs.project_config.defaults - target_values: tuple[str | None, ...] = ( - (target.database, target.schema, target.loader_schema) if target is not None else () - ) - default_values: tuple[str | None, ...] = ( - defaults.database, - defaults.schema, - defaults.seed_database, - defaults.seed_schema, - ) - seed_values: list[str | None] = [] - for seed_input in inputs.seed_inputs: - seed_values.extend((seed_input.schema_entry.database, seed_input.schema_entry.schema)) - row: ProjectRequestRow = ( - dialect or "generic", - (target.database, target.schema, target.loader_schema) if target is not None else None, - (defaults.database, defaults.schema, defaults.seed_database, defaults.seed_schema), - [ - _variable_row(name=name, value=value, render=render, index=index) - for index, ((name, value), render) in enumerate( - zip(inputs.effective_vars.items(), renders, strict=True) - ) - ], - _environment_rows((*target_values, *default_values, *seed_values)), - [ - ( - model_input.model_file.file_path.stem, - str(model_input.model_file.relative_path.parent), - _tag_rows(model_input.config.values.get("tags")), - _reference_rows(model_input.references), - _syntax_check_rows(checks), - ) - for model_input, checks in zip(inputs.model_inputs, syntax_checks, strict=True) - ], - [source_input.assembly_row for source_input in inputs.source_inputs], - [ - ( - seed_input.schema_entry.name, - list(seed_input.schema_entry.tags), - seed_input.schema_entry.database, - seed_input.schema_entry.schema, - ) - for seed_input in inputs.seed_inputs - ], - [function_input.assembly_row for function_input in inputs.sql_function_inputs], - [audit_input.assembly_row for audit_input in inputs.audit_inputs], - ) - return row, [render if isinstance(render, Exception) else None for render in renders] - - -def _tag_rows(value: object) -> list[str]: - """Model config tags as the project graph indexes them.""" - - if isinstance(value, list | tuple): - return [str(item) for item in value] - return [] - - -def _environment_rows(values: Iterable[str | None]) -> list[tuple[str, str | None]]: - """Python's own `os.environ` lookup of every name an `ENV:` template may read.""" - - return [ - (name, os.environ.get(name)) for name in _native.environment_template_names(list(values)) - ] - - -def _variable_row( - *, name: str, value: object, render: tuple[str, str] | Exception | None, index: int -) -> VariableRow: - if value is None: - return (name, NONE_VARIABLE, "", None) - if type(value) is bool: - return (name, BOOLEAN_VARIABLE, TRUE_TEXT if value else FALSE_TEXT, None) - if type(value) is str: - return (name, TEXT_VARIABLE, value, None) - if type(value) is int and NATIVE_INT_MIN <= value <= NATIVE_INT_MAX: - return (name, TEXT_VARIABLE, str(value), None) - held: tuple[str, str] = (RENDER_RAISES, str(index)) - return (name, OPAQUE_VARIABLE, str(value), held if isinstance(render, Exception) else render) - - -def _opaque_render(*, value: object) -> tuple[str, str] | Exception | None: - """`render_project_var_text` for any label: text, the message after the label, or the error.""" - - if value is None or type(value) in (bool, str): - return None - if type(value) is int and NATIVE_INT_MIN <= value <= NATIVE_INT_MAX: - return None - try: - text: str = render_project_var_text(value=value, label=RENDER_LABEL) - except ValueError as error: - message: str = str(error) - if message.startswith(RENDER_LABEL): - return (RENDER_REJECTED, message[len(RENDER_LABEL) :]) - return error - except Exception as error: - return error - return (RENDERED_TEXT, text) - - -def _reference_rows(references: tuple[CompileSqlReference, ...]) -> list[ReferenceRow]: - return [ - (str(reference.ref_kind), reference.ref_name, reference.ref_package) - for reference in references - ] - - -def _syntax_check_rows( - checks: tuple[tuple[str, dict[str, str] | None], ...], -) -> list[SyntaxCheckRow]: - return [(sql, list((placeholders or {}).items())) for sql, placeholders in checks] diff --git a/src/sqlbuild/compiler/project_assembly/_helpers/resources.py b/src/sqlbuild/compiler/project_assembly/_helpers/resources.py deleted file mode 100644 index 7174dd5677..0000000000 --- a/src/sqlbuild/compiler/project_assembly/_helpers/resources.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Materialise the native project assembly result as the objects Python's assembly builds.""" - -from __future__ import annotations - -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.project_assembly.constants import ( - COMPILE_INPUT_FAILURE, - ENVIRONMENT_READ, - HELD_FAILURE, - VALUE_FAILURE, -) -from sqlbuild.compiler.project_assembly.models import NativeProjectResources, NativeResourceFailure -from sqlbuild.compiler.project_assembly.types import ( - FailureRow, - ProjectResourcesRow, -) -from sqlbuild.compiler.sql_analysis.constants import NATIVE_FAILURE_PREFIX - - -def project_resources( - *, inputs: CompileInputsView, row: ProjectResourcesRow, held: list[Exception | None] -) -> NativeProjectResources: - """Return the resource facts the native row describes, in input order, up to the failure.""" - - ( - model_deps, - source_namespaces, - seed_namespaces, - function_deps, - audit_deps, - reads, - syntax, - failure, - ) = row - _ = [_replay_read(kind=kind, name=name) for kind, name in reads] - return NativeProjectResources( - model_deps=tuple(model_deps), - model_syntax_valid=tuple(syntax), - source_namespaces=tuple(source_namespaces), - seed_namespaces=tuple(seed_namespaces), - function_deps=tuple(function_deps), - audit_scope_deps=tuple(audit_deps), - failure=None if failure is None else _failure(row=failure, held=held), - ) - - -def _failure(*, row: FailureRow, held: list[Exception | None]) -> NativeResourceFailure: - slot, index, kind, message, help_text, held_index = row - error: Exception - if kind == COMPILE_INPUT_FAILURE: - error = CompileInputError(message, help=help_text) - elif kind == VALUE_FAILURE: - error = ValueError(message) - elif kind == HELD_FAILURE and held_index is not None and held[held_index] is not None: - error = cast(Exception, held[held_index]) - else: - raise _native.NativeCompilerError( - f"{NATIVE_FAILURE_PREFIX}native project assembly: unknown failure kind '{kind}'" - ) - return NativeResourceFailure(slot=slot, index=index, error=error) - - -def _replay_read(*, kind: str, name: str) -> None: - if kind == ENVIRONMENT_READ: - COMPILE_INPUT_READS.environment_read(name) - else: - COMPILE_INPUT_READS.context_read(name) diff --git a/src/sqlbuild/compiler/project_assembly/constants.py b/src/sqlbuild/compiler/project_assembly/constants.py deleted file mode 100644 index 78c8ec8154..0000000000 --- a/src/sqlbuild/compiler/project_assembly/constants.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Native project assembly request and failure kinds.""" - -TEXT_VARIABLE: str = "text" -BOOLEAN_VARIABLE: str = "bool" -NONE_VARIABLE: str = "none" -OPAQUE_VARIABLE: str = "opaque" -RENDERED_TEXT: str = "text" -RENDER_REJECTED: str = "rejected" -RENDER_RAISES: str = "raises" -RENDER_LABEL: str = "\x00" -COMPILE_INPUT_FAILURE: str = "compile_input" -VALUE_FAILURE: str = "value" -HELD_FAILURE: str = "held" -SOURCE_SLOT: str = "source" -SEED_SLOT: str = "seed" -AUDIT_SLOT: str = "audit" -TRUE_TEXT: str = "1" -FALSE_TEXT: str = "0" -NATIVE_INT_MIN: int = -(2**63) -NATIVE_INT_MAX: int = 2**63 - 1 -ENVIRONMENT_READ: str = "env" diff --git a/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py b/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py deleted file mode 100644 index a16528e4e1..0000000000 --- a/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Assemble the compiled project's resource facts natively.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.project_assembly._helpers.request import project_request -from sqlbuild.compiler.project_assembly._helpers.resources import project_resources -from sqlbuild.compiler.project_assembly.models import NativeProjectResources -from sqlbuild.compiler.project_assembly.types import ProjectRequestRow, ProjectResourcesRow - - -def assemble_native_project_resources( - *, - inputs: CompileInputsView, - dialect: str | None, - syntax_checks: tuple[tuple[tuple[str, dict[str, str] | None], ...], ...], -) -> NativeProjectResources: - """Return the resource facts and the resource Python's assembly raises at, if any.""" - - request: ProjectRequestRow - held: list[Exception | None] - request, held = project_request(inputs=inputs, dialect=dialect, syntax_checks=syntax_checks) - row: ProjectResourcesRow = _native.assemble_project_resource_facts(request) - return project_resources(inputs=inputs, row=row, held=held) diff --git a/src/sqlbuild/compiler/project_assembly/types.py b/src/sqlbuild/compiler/project_assembly/types.py index afcec02958..65e981eb20 100644 --- a/src/sqlbuild/compiler/project_assembly/types.py +++ b/src/sqlbuild/compiler/project_assembly/types.py @@ -2,38 +2,5 @@ from __future__ import annotations -type ReferenceRow = tuple[str, str, str | None] -type SyntaxCheckRow = tuple[str, list[tuple[str, str]]] -type VariableRow = tuple[str, str, str, tuple[str, str] | None] -type FailureRow = tuple[str, int, str, str, str | None, int | None] -type ModelRow = tuple[str, str, list[str], list[ReferenceRow], list[SyntaxCheckRow]] -type SourceRow = tuple[str, bool, str | None, str | None] -type SeedRow = tuple[str, list[str], str | None, str | None] -type FunctionRow = tuple[str, str, list[str], list[ReferenceRow]] -type AuditRow = tuple[list[ReferenceRow], tuple[str, str] | None] -type TargetRow = tuple[str | None, str | None, str | None] -type DefaultsRow = tuple[str | None, str | None, str | None, str | None] -type ProjectRequestRow = tuple[ - str, - TargetRow | None, - DefaultsRow, - list[VariableRow], - list[tuple[str, str | None]], - list[ModelRow], - list[SourceRow], - list[SeedRow], - list[FunctionRow], - list[AuditRow], -] type ObjectKeyRow = tuple[str, str] type NamespaceRow = tuple[str | None, str | None, str | None, str | None] -type ProjectResourcesRow = tuple[ - list[list[ObjectKeyRow]], - list[tuple[str | None, str | None] | None], - list[NamespaceRow], - list[list[ObjectKeyRow]], - list[list[ObjectKeyRow]], - list[tuple[str, str]], - list[bool], - FailureRow | None, -] diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/run_lifecycle.py b/src/sqlbuild/compiler/python_nodes/_helpers/run_lifecycle.py deleted file mode 100644 index ecc3aa83e2..0000000000 --- a/src/sqlbuild/compiler/python_nodes/_helpers/run_lifecycle.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Lifecycle classification for build-style SQL/Python selections.""" - -from __future__ import annotations - -from sqlbuild._native import transitive_closure -from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonLoaderMetadata, - PythonNodeGraph, - PythonSqlRunLifecyclePlan, - PythonSqlRunSelection, -) -from sqlbuild.compiler.python_nodes.types import PythonNodeKind - - -def build_python_sql_run_lifecycle_plan( - *, selection: PythonSqlRunSelection, python_graph: PythonNodeGraph -) -> PythonSqlRunLifecyclePlan: - """Classify selected run nodes into lifecycle-aware execution regions.""" - - selected_python_names: frozenset[str] = selection.python_node_names - selected_loader_names: frozenset[str] = frozenset( - name - for name in selected_python_names - if python_graph.nodes_by_name[name].kind == PythonNodeKind.LOADER - if not _is_external_loader(node_name=name, python_graph=python_graph) - ) - ingress_upstream_names: set[str] = set() - for loader_name in selected_loader_names: - for upstream_name in transitive_closure( - start=loader_name, edges=python_graph.upstream_deps - ): - if upstream_name not in selected_python_names: - continue - if python_graph.nodes_by_name[upstream_name].kind in { - PythonNodeKind.TASK, - PythonNodeKind.ASSET, - PythonNodeKind.LOADER, - }: - ingress_upstream_names.add(upstream_name) - ingress_names: frozenset[str] = selected_loader_names | frozenset(ingress_upstream_names) - read_side_names: frozenset[str] = frozenset( - name - for name in selected_python_names - ingress_names - if python_graph.nodes_by_name[name].kind in {PythonNodeKind.TASK, PythonNodeKind.ASSET} - ) - return PythonSqlRunLifecyclePlan( - ingress_python_node_names=ingress_names, - ingress_loader_names=selected_loader_names, - read_side_python_node_names=read_side_names, - ) - - -def _is_external_loader(*, node_name: str, python_graph: PythonNodeGraph) -> bool: - loader: DiscoveredPythonLoaderMetadata | None = python_graph.nodes_by_name[node_name].loader - return loader is not None and loader.connection_mode == LoaderConnectionMode.EXTERNAL diff --git a/src/sqlbuild/compiler/python_nodes/main/_run_lifecycle.py b/src/sqlbuild/compiler/python_nodes/main/_run_lifecycle.py deleted file mode 100644 index c6b248a5a8..0000000000 --- a/src/sqlbuild/compiler/python_nodes/main/_run_lifecycle.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Public entrypoint for run-command Python/SQL lifecycle planning.""" - -from __future__ import annotations - -from sqlbuild.compiler.python_nodes._helpers.run_lifecycle import ( - build_python_sql_run_lifecycle_plan, -) -from sqlbuild.compiler.python_nodes.models import ( - PythonNodeGraph, - PythonSqlRunLifecyclePlan, - PythonSqlRunSelection, -) - - -def build_python_sql_run_lifecycle( - *, selection: PythonSqlRunSelection, python_graph: PythonNodeGraph -) -> PythonSqlRunLifecyclePlan: - """Classify selected run nodes into lifecycle-aware execution regions.""" - - return build_python_sql_run_lifecycle_plan( - selection=selection, - python_graph=python_graph, - ) diff --git a/src/sqlbuild/compiler/references/_helpers/__init__.py b/src/sqlbuild/compiler/references/_helpers/__init__.py deleted file mode 100644 index 2d2301793d..0000000000 --- a/src/sqlbuild/compiler/references/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Internal SQL reference helpers.""" diff --git a/src/sqlbuild/compiler/references/_helpers/patterns.py b/src/sqlbuild/compiler/references/_helpers/patterns.py deleted file mode 100644 index c50df5aabb..0000000000 --- a/src/sqlbuild/compiler/references/_helpers/patterns.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Regex helpers for SQLBuild reference function syntax.""" - -from __future__ import annotations - -import re - -from sqlbuild.compiler.references.types import SqlReferenceKind - - -def reference_call_prefix_pattern_text(kind: SqlReferenceKind) -> str: - return rf"{re.escape(kind.function_name)}\(" - - -def quoted_reference_call_pattern_text(kind: SqlReferenceKind) -> str: - return rf'{reference_call_prefix_pattern_text(kind)}"([^"]+)"\)' - - -def quoted_reference_call_pattern(kind: SqlReferenceKind) -> re.Pattern[str]: - return re.compile(quoted_reference_call_pattern_text(kind)) diff --git a/src/sqlbuild/compiler/references/constants.py b/src/sqlbuild/compiler/references/constants.py index 494af6394e..68c894a12a 100644 --- a/src/sqlbuild/compiler/references/constants.py +++ b/src/sqlbuild/compiler/references/constants.py @@ -4,9 +4,19 @@ import re -from sqlbuild.compiler.references._helpers.patterns import quoted_reference_call_pattern from sqlbuild.compiler.references.types import SqlReferenceKind -REF_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.REF) -SEED_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.SEED) -SOURCE_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.SOURCE) +REF_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.REF.function_name)}\(" +SEED_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.SEED.function_name)}\(" +SOURCE_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.SOURCE.function_name)}\(" +DBT_REF_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.DBT_REF.function_name)}\(" +UDF_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.UDF.function_name)}\(" +TABLE_FUNCTION_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.TABLE_FUNCTION.function_name)}\(" +_QUOTED_NAME_ARGUMENT: str = r'"([^"]+)"\)' +REF_PATTERN: re.Pattern[str] = re.compile(REF_CALL_PREFIX + _QUOTED_NAME_ARGUMENT) +SEED_PATTERN: re.Pattern[str] = re.compile(SEED_CALL_PREFIX + _QUOTED_NAME_ARGUMENT) +SOURCE_PATTERN: re.Pattern[str] = re.compile(SOURCE_CALL_PREFIX + _QUOTED_NAME_ARGUMENT) +UDF_PATTERN: re.Pattern[str] = re.compile(UDF_CALL_PREFIX + _QUOTED_NAME_ARGUMENT) +TABLE_FUNCTION_PATTERN: re.Pattern[str] = re.compile( + TABLE_FUNCTION_CALL_PREFIX + _QUOTED_NAME_ARGUMENT +) diff --git a/src/sqlbuild/compiler/references/main/__init__.py b/src/sqlbuild/compiler/references/main/__init__.py deleted file mode 100644 index 0045272af3..0000000000 --- a/src/sqlbuild/compiler/references/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Public SQL reference entries.""" diff --git a/src/sqlbuild/compiler/references/main/_quoted_reference_call_pattern.py b/src/sqlbuild/compiler/references/main/_quoted_reference_call_pattern.py deleted file mode 100644 index 3c8ef9af89..0000000000 --- a/src/sqlbuild/compiler/references/main/_quoted_reference_call_pattern.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public quoted SQL reference pattern entry.""" - -from __future__ import annotations - -import re - -from sqlbuild.compiler.references._helpers.patterns import ( - quoted_reference_call_pattern as _quoted_reference_call_pattern, -) -from sqlbuild.compiler.references.types import SqlReferenceKind - - -def quoted_reference_call_pattern(kind: SqlReferenceKind) -> re.Pattern[str]: - return _quoted_reference_call_pattern(kind) diff --git a/src/sqlbuild/compiler/references/main/_reference_call_prefix_pattern_text.py b/src/sqlbuild/compiler/references/main/_reference_call_prefix_pattern_text.py deleted file mode 100644 index 1f9b151d12..0000000000 --- a/src/sqlbuild/compiler/references/main/_reference_call_prefix_pattern_text.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public SQL reference prefix pattern entry.""" - -from __future__ import annotations - -from sqlbuild.compiler.references._helpers.patterns import ( - reference_call_prefix_pattern_text as _reference_call_prefix_pattern_text, -) -from sqlbuild.compiler.references.types import SqlReferenceKind - - -def reference_call_prefix_pattern_text(kind: SqlReferenceKind) -> str: - return _reference_call_prefix_pattern_text(kind) diff --git a/src/sqlbuild/compiler/resource_names/main/__init__.py b/src/sqlbuild/compiler/resource_names/main/__init__.py deleted file mode 100644 index 2afe5c8342..0000000000 --- a/src/sqlbuild/compiler/resource_names/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Resource name validation entrypoints.""" diff --git a/src/sqlbuild/compiler/resource_names/main/_validate_resource_identity.py b/src/sqlbuild/compiler/resource_names/main/_validate_resource_identity.py deleted file mode 100644 index 9181d26aa2..0000000000 --- a/src/sqlbuild/compiler/resource_names/main/_validate_resource_identity.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Canonical resource identity validation entrypoint.""" - -from __future__ import annotations - -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError - - -def validate_resource_identity( - *, name: str, kind: str, path: Path, private_identity: bool = False -) -> None: - """Reject one identity unless it uses canonical lowercase snake_case.""" - - failure: tuple[str, str] | None = _native.resource_identity_suggestion(name, private_identity) - if failure is None: - return - corrected_name, help_text = failure - raise ResourceIdentityError( - f"Invalid {kind} identity '{name}' in {path}; use snake_case '{corrected_name}'", - help=help_text, - ) diff --git a/src/sqlbuild/compiler/resource_names/main/function_node_type.py b/src/sqlbuild/compiler/resource_names/main/function_node_type.py deleted file mode 100644 index 6caf6e2eca..0000000000 --- a/src/sqlbuild/compiler/resource_names/main/function_node_type.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Resolve compiled resource type for SQL functions.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledResourceType - - -def _function_resource_type(*, return_columns: tuple[object, ...]) -> CompiledResourceType: - """Return the compiled resource type for a SQL function shape.""" - - return CompiledResourceType.TABLE_FN if return_columns else CompiledResourceType.UDF - - -def function_node_type(*, return_columns: tuple[object, ...]) -> str: - """Return the public node type for a SQL function shape.""" - - return _function_resource_type(return_columns=return_columns).value diff --git a/src/sqlbuild/compiler/scopes/_helpers/__init__.py b/src/sqlbuild/compiler/scopes/_helpers/__init__.py deleted file mode 100644 index 8b6d28cf22..0000000000 --- a/src/sqlbuild/compiler/scopes/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Internal compiler scope helpers.""" diff --git a/src/sqlbuild/compiler/scopes/_helpers/paths.py b/src/sqlbuild/compiler/scopes/_helpers/paths.py deleted file mode 100644 index b63d0b00e8..0000000000 --- a/src/sqlbuild/compiler/scopes/_helpers/paths.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Project-relative scope path normalization, answered by the native scope paths.""" - -from __future__ import annotations - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError - - -def normalize_path(*, path: str | PurePath) -> str: - """The canonical project-relative POSIX path; absolute or escaping paths raise.""" - - normalized, error = _native.scope_normalize_path(str(path)) - if normalized is None: - raise InvalidScopePathError(error or "") - return normalized diff --git a/src/sqlbuild/compiler/scopes/main/__init__.py b/src/sqlbuild/compiler/scopes/main/__init__.py deleted file mode 100644 index 97dd363eac..0000000000 --- a/src/sqlbuild/compiler/scopes/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compiler scope entry modules.""" diff --git a/src/sqlbuild/compiler/scopes/main/_build_scope_index.py b/src/sqlbuild/compiler/scopes/main/_build_scope_index.py deleted file mode 100644 index c1a3187392..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_build_scope_index.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Build the canonical static scope index from discovered project inputs.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.compiler.compile.models import LoadedMacro, StaticMacroExport -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index -from sqlbuild.compiler.scopes.models import ScopeIndex - - -def build_scope_index( - *, - discovered_inputs: DiscoveredProjectInputs, - loaded_macros: Mapping[str, LoadedMacro] | None = None, - static_macros: tuple[StaticMacroExport, ...] = (), -) -> ScopeIndex: - """Project discovery facts into a deterministic, partially valid scope index.""" - - return open_native_scope_index( - discovered_inputs=discovered_inputs, - loaded_macros=loaded_macros or {}, - static_macros=static_macros, - ).index diff --git a/src/sqlbuild/compiler/scopes/main/_declaration_lexical_path.py b/src/sqlbuild/compiler/scopes/main/_declaration_lexical_path.py deleted file mode 100644 index 7856678e28..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_declaration_lexical_path.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Lexical location of a declaration file for visibility resolution.""" - -from __future__ import annotations - -from sqlbuild.compiler.scopes.constants import ( - CURRENT_PATH_COMPONENT, - DECLARATION_LEXICAL_STEM, - PATH_SEPARATOR, -) -from sqlbuild.compiler.scopes.models import DeclarationRecord - - -def declaration_lexical_path(*, record: DeclarationRecord) -> str: - """Return the authored path whose folder controls what a declaration file can use.""" - - return ( - f"{record.owning_path or CURRENT_PATH_COMPONENT}{PATH_SEPARATOR}{DECLARATION_LEXICAL_STEM}" - ) diff --git a/src/sqlbuild/compiler/scopes/main/_get_placement_validated_scope_index.py b/src/sqlbuild/compiler/scopes/main/_get_placement_validated_scope_index.py deleted file mode 100644 index 08cd685100..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_get_placement_validated_scope_index.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Public entrypoint for declaration placement validation.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ScopeIndex, UsageRecord - - -def get_placement_validated_scope_index( - *, - index: ScopeIndex, - enforce_placement: bool = True, - compile_usages: tuple[UsageRecord, ...] | None = None, -) -> ScopeIndex: - """Return complete usage and placement diagnostics, merging any collected compile usages.""" - - return _native.scope_placement_validated_index(index, enforce_placement, compile_usages) diff --git a/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py b/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py deleted file mode 100644 index 2679b5344f..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Build the static scope index natively.""" - -from __future__ import annotations - -from collections.abc import Mapping - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import LoadedMacro, StaticMacroExport -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs - - -def open_native_scope_index( - *, - discovered_inputs: DiscoveredProjectInputs, - loaded_macros: Mapping[str, LoadedMacro], - static_macros: tuple[StaticMacroExport, ...] = (), -) -> _native.NativeScopeIndex: - """Return the natively built index; discovery's normalized paths always index.""" - - return _native.open_native_scope_index(discovered_inputs, loaded_macros, static_macros) diff --git a/src/sqlbuild/compiler/scopes/main/_query_scope_target.py b/src/sqlbuild/compiler/scopes/main/_query_scope_target.py deleted file mode 100644 index d1ab0243c3..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_query_scope_target.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Resolve what one scope target names.""" - -from __future__ import annotations - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - ResourceIdentity, - ScopeLookup, - ScopeTargetQuery, -) - - -def query_scope_target( - *, lookup: ScopeLookup, target: ResourceIdentity | DeclarationIdentity | str | PurePath -) -> ScopeTargetQuery: - """The resources or declarations `target` names; invalid text raises the scope error.""" - - return _native.scope_query_target(lookup, target) diff --git a/src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py b/src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py deleted file mode 100644 index 427b4814f6..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Resolve static declaration visibility without per-declaration inaccessible reasons.""" - -from __future__ import annotations - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import DeclarationVisibility, ResourceIdentity, ScopeLookup - - -def resolve_scope_declaration_visibility( - *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath -) -> DeclarationVisibility: - """Return visible facts and inaccessible declaration identities for one target.""" - - return _native.scope_resolve_visibility(lookup, target, False) diff --git a/src/sqlbuild/compiler/scopes/main/_resolve_scope_path_visibility.py b/src/sqlbuild/compiler/scopes/main/_resolve_scope_path_visibility.py deleted file mode 100644 index acb0dbf959..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_resolve_scope_path_visibility.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Resolve declaration visibility for any authored project-relative path.""" - -from __future__ import annotations - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import DeclarationRecord, ScopeLookup - - -def resolve_scope_path_visibility( - *, lookup: ScopeLookup, path: str | PurePath -) -> tuple[tuple[DeclarationRecord, ...], tuple[DeclarationRecord, ...]]: - """Return visible and inaccessible declarations for an authored path.""" - - return _native.scope_resolve_path_visibility(lookup, path) diff --git a/src/sqlbuild/compiler/scopes/main/_resolve_scope_visibility.py b/src/sqlbuild/compiler/scopes/main/_resolve_scope_visibility.py deleted file mode 100644 index 098231c4dc..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_resolve_scope_visibility.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Resolve canonical static declaration visibility for one target.""" - -from __future__ import annotations - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ResourceIdentity, ScopeLookup, VisibilityResolution - - -def resolve_scope_visibility( - *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath -) -> VisibilityResolution: - """Return visible and inaccessible facts, or an explicit unknown target result.""" - - return _native.scope_resolve_visibility(lookup, target, True) diff --git a/src/sqlbuild/compiler/scopes/main/_validate_scope_index.py b/src/sqlbuild/compiler/scopes/main/_validate_scope_index.py deleted file mode 100644 index 2052f0b25c..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_validate_scope_index.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Strict validation for an already-built, fact-preserving scope index.""" - -from __future__ import annotations - -from sqlbuild.compiler.scopes.exceptions import ScopeValidationError -from sqlbuild.compiler.scopes.models import ScopeDiagnostic, ScopeIndex -from sqlbuild.compiler.scopes.types import DiagnosticSeverity - - -def validate_scope_index(*, index: ScopeIndex) -> None: - """Raise one aggregate compiler error when the index has error diagnostics.""" - - errors: tuple[ScopeDiagnostic, ...] = tuple( - item for item in index.diagnostics if item.severity is DiagnosticSeverity.ERROR - ) - if errors: - raise ScopeValidationError(errors) diff --git a/src/sqlbuild/compiler/scopes/main/browse_scope_folders.py b/src/sqlbuild/compiler/scopes/main/browse_scope_folders.py deleted file mode 100644 index c91b4f0f26..0000000000 --- a/src/sqlbuild/compiler/scopes/main/browse_scope_folders.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Public declaration folder browse operation.""" - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ResourceIdentity, ScopeBrowseResult, ScopeLookup - - -def browse_scope_folders( - *, - lookup: ScopeLookup, - folder: str = ".", - target: str | PurePath | ResourceIdentity | None = None, - target_is_prospective: bool = False, -) -> ScopeBrowseResult: - """Browse direct declaration-definition child folders only.""" - - return _native.scope_browse(lookup, (folder, target, target_is_prospective)) diff --git a/src/sqlbuild/compiler/scopes/main/build_scope_lookup.py b/src/sqlbuild/compiler/scopes/main/build_scope_lookup.py deleted file mode 100644 index d96e802679..0000000000 --- a/src/sqlbuild/compiler/scopes/main/build_scope_lookup.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Build immutable lookup indexes over canonical scope facts.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ScopeIndex, ScopeLookup - - -def build_scope_lookup(*, index: ScopeIndex) -> ScopeLookup: - """Canonicalize scope facts and build deeply immutable lookup mappings.""" - - return _native.build_scope_lookup(index) diff --git a/src/sqlbuild/compiler/scopes/main/explain_scope_declaration.py b/src/sqlbuild/compiler/scopes/main/explain_scope_declaration.py deleted file mode 100644 index f2f339fa40..0000000000 --- a/src/sqlbuild/compiler/scopes/main/explain_scope_declaration.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Public detailed declaration explanation operation.""" - -from __future__ import annotations - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ( - DeclarationExplanation, - DeclarationIdentity, - ResourceIdentity, - ScopeDiagnostic, - ScopeLookup, -) - - -def explain_scope_declaration( - *, - lookup: ScopeLookup, - declaration: str | DeclarationIdentity, - target: str | PurePath | ResourceIdentity | None = None, - at: str | PurePath | None = None, - directory: bool = False, -) -> tuple[DeclarationExplanation, tuple[ScopeDiagnostic, ...]]: - """Explain a qualified declaration in an optional resource/path context.""" - - return _native.scope_explain(lookup, (declaration, target, at, directory)) diff --git a/src/sqlbuild/compiler/scopes/main/list_scope_declarations.py b/src/sqlbuild/compiler/scopes/main/list_scope_declarations.py deleted file mode 100644 index e46c688f92..0000000000 --- a/src/sqlbuild/compiler/scopes/main/list_scope_declarations.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Public recursive declaration folder list operation.""" - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ( - ResourceIdentity, - ScopeListResult, - ScopeLookup, - ScopeReportFilters, -) - - -def list_scope_declarations( - *, - lookup: ScopeLookup, - folder: str, - filters: ScopeReportFilters | None = None, - target: str | PurePath | ResourceIdentity | None = None, - target_is_prospective: bool = False, -) -> ScopeListResult: - """List declarations recursively beneath one browsed folder.""" - - return _native.scope_list(lookup, (folder, filters, target, target_is_prospective)) diff --git a/src/sqlbuild/compiler/scopes/main/preview_scope_move.py b/src/sqlbuild/compiler/scopes/main/preview_scope_move.py deleted file mode 100644 index 6b4c3a999c..0000000000 --- a/src/sqlbuild/compiler/scopes/main/preview_scope_move.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Public pure resource move preview operation.""" - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ( - MovePreview, - ResourceIdentity, - ScopeDiagnostic, - ScopeLookup, -) - - -def preview_scope_move( - *, lookup: ScopeLookup, resource: str | ResourceIdentity, destination: str | PurePath -) -> tuple[MovePreview | None, tuple[ScopeDiagnostic, ...]]: - """Preview declaration visibility changes without mutating compiler facts.""" - - return _native.scope_move_preview(lookup, resource, destination) diff --git a/src/sqlbuild/compiler/scopes/main/query_scope_report.py b/src/sqlbuild/compiler/scopes/main/query_scope_report.py deleted file mode 100644 index 59ccc704a8..0000000000 --- a/src/sqlbuild/compiler/scopes/main/query_scope_report.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Public pure scope report operation.""" - -from __future__ import annotations - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - ResourceIdentity, - ScopeLookup, - ScopeReport, - ScopeReportFilters, -) - - -def query_scope_report( - *, - lookup: ScopeLookup, - target: str | PurePath | ResourceIdentity | DeclarationIdentity | None = None, - at: str | PurePath | None = None, - directory: bool = False, - filters: ScopeReportFilters | None = None, -) -> ScopeReport: - """Query immutable compiler-owned scope facts without filesystem access.""" - - return _native.scope_report(lookup, (target, at, directory, filters)) diff --git a/src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py b/src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py deleted file mode 100644 index a962bc5632..0000000000 --- a/src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public declaration relocation for a resource move.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import DeclarationRecord, ResourceIdentity, ScopeIndex - - -def relocate_declarations_for_move( - *, index: ScopeIndex, resource: ResourceIdentity, destination: str -) -> tuple[DeclarationRecord, ...] | None: - """Return declarations that must move, at their new paths, when a resource moves.""" - - return _native.scope_relocated_declarations(index, resource, destination) diff --git a/src/sqlbuild/compiler/scopes/main/scope_metadata.py b/src/sqlbuild/compiler/scopes/main/scope_metadata.py deleted file mode 100644 index c6af02e388..0000000000 --- a/src/sqlbuild/compiler/scopes/main/scope_metadata.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public value-safe projection and serialization of canonical compiler scope facts.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ScopeIndex -from sqlbuild.compiler.scopes.types import JsonValue - - -def scope_metadata_projection(*, index: ScopeIndex) -> dict[str, JsonValue]: - """Return deterministic, value-free scope metadata for external consumers.""" - - return _native.scope_metadata_projection(index) diff --git a/src/sqlbuild/compiler/scopes/main/serialize_scope_report.py b/src/sqlbuild/compiler/scopes/main/serialize_scope_report.py deleted file mode 100644 index 978ac7a1d5..0000000000 --- a/src/sqlbuild/compiler/scopes/main/serialize_scope_report.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Public canonical scope report serialization operation.""" - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ScopeBrowseResult, ScopeListResult, ScopeReport - - -def serialize_scope_report(*, report: ScopeReport | ScopeBrowseResult | ScopeListResult) -> str: - """Return deterministic schema-version-one JSON for any scope query result.""" - - return _native.serialize_scope_report(report) diff --git a/src/sqlbuild/executor/audit_results/_helpers/sql.py b/src/sqlbuild/executor/audit_results/_helpers/sql.py index 296d7a1a36..986206bad2 100644 --- a/src/sqlbuild/executor/audit_results/_helpers/sql.py +++ b/src/sqlbuild/executor/audit_results/_helpers/sql.py @@ -4,6 +4,7 @@ from collections.abc import Callable, Sequence +from sqlbuild._native import render_state_sql_literal from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( render_state_table_create_sql, @@ -54,7 +55,6 @@ COLUMN_VIOLATION_COUNT, ) from sqlbuild.executor.audit_results.models import AuditResultRecord -from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal _REQUIRED_COLUMNS: frozenset[str] = frozenset( { diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index 92972ec941..1e41b5847a 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -4,15 +4,15 @@ from collections.abc import Callable, Mapping -from sqlbuild._native import assert_no_unresolved_sql_markers +from sqlbuild._native import ( + assert_no_unresolved_sql_markers, + replaces_incremental_relation, + resolve_cursor_intrinsics, +) from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL -from sqlbuild.compiler.planner.main.changes.relation_replacement import ( - replaces_incremental_relation, -) from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.compiler.planner.types import ( IncrementalMode, diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 49e8dfc4cc..220e1bf020 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -108,7 +108,7 @@ from sqlbuild.executor.seed.main._execute import execute_seed from sqlbuild.executor.testing.constants import SQL_TEST_ENTRY_MISSING_CODE from sqlbuild.executor.testing.main._execute import execute_sql_test -from sqlbuild.executor.testing.main.resource_id import sql_test_resource_id +from sqlbuild.executor.testing.main._resource_id import sql_test_resource_id from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome from sqlbuild.microbatches.classes.direct_store import ( diff --git a/src/sqlbuild/executor/functions/main/_execute.py b/src/sqlbuild/executor/functions/main/_execute.py index 1daeda2723..b90c1b264e 100644 --- a/src/sqlbuild/executor/functions/main/_execute.py +++ b/src/sqlbuild/executor/functions/main/_execute.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( compute_function_definition_hash, + function_node_type, write_fingerprint, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -16,7 +17,6 @@ from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import FunctionPlanEntry -from sqlbuild.compiler.resource_names.main.function_node_type import function_node_type from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.errors.contracts.main.error_code import error_code from sqlbuild.errors.contracts.main.error_help import error_help diff --git a/src/sqlbuild/executor/janitor_events/_helpers/sql.py b/src/sqlbuild/executor/janitor_events/_helpers/sql.py index a3f3a3202e..5d85cae77f 100644 --- a/src/sqlbuild/executor/janitor_events/_helpers/sql.py +++ b/src/sqlbuild/executor/janitor_events/_helpers/sql.py @@ -4,6 +4,7 @@ from collections.abc import Callable +from sqlbuild._native import render_state_sql_literal from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( render_state_table_create_sql, @@ -29,7 +30,6 @@ JANITOR_EVENTS_TABLE_NAME, ) from sqlbuild.executor.janitor_events.models import JanitorEventRecord -from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal def build_qualified_table_name( diff --git a/src/sqlbuild/executor/node_results/_helpers/sql.py b/src/sqlbuild/executor/node_results/_helpers/sql.py index be338e043f..d8669e4ee9 100644 --- a/src/sqlbuild/executor/node_results/_helpers/sql.py +++ b/src/sqlbuild/executor/node_results/_helpers/sql.py @@ -4,6 +4,7 @@ from collections.abc import Callable +from sqlbuild._native import render_state_sql_literal from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( render_state_table_create_sql, @@ -27,7 +28,6 @@ NODE_RESULTS_TABLE_NAME, ) from sqlbuild.executor.node_results.models import NodeResultQuery, NodeResultRecord -from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal _REQUIRED_NODE_RESULT_COLUMNS: frozenset[str] = frozenset( { diff --git a/src/sqlbuild/executor/pipeline/_helpers/scenario.py b/src/sqlbuild/executor/pipeline/_helpers/scenario.py index 1ecac28fe1..a7efed0c55 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/scenario.py +++ b/src/sqlbuild/executor/pipeline/_helpers/scenario.py @@ -18,6 +18,7 @@ from sqlbuild._native import ( NativeProgressProjector, current_native_progress_projector, + scenario_local_type_overrides_for_dialect, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import TablePromotionMode @@ -64,9 +65,6 @@ from sqlbuild.runtime.observability.classes.resource_attempt_lifecycle import ( ResourceAttemptLifecycle, ) -from sqlbuild.spec.contracts.main.scenario_local_type_overrides_for_dialect import ( - scenario_local_type_overrides_for_dialect, -) _DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") _STOP_POLL_SECONDS: float = 0.05 diff --git a/src/sqlbuild/executor/pipeline/_helpers/testing.py b/src/sqlbuild/executor/pipeline/_helpers/testing.py index 152c03702c..bf5ebd668b 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/testing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/testing.py @@ -31,7 +31,7 @@ from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.constants import SQL_TEST_EXECUTION_ERROR_CODE from sqlbuild.executor.testing.main._execute import execute_sql_test -from sqlbuild.executor.testing.main.resource_id import sql_test_resource_id +from sqlbuild.executor.testing.main._resource_id import sql_test_resource_id from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.executor.testing.types import SqlTestOutcome from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index c5c1454d21..8d0eba4ad8 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -16,6 +16,7 @@ compute_query_hash, cursor_bound_display, inclusive_cursor_end, + microbatch_limit_warning, resolve_effective_microbatch_batch_size, ) from sqlbuild._native import cursor_cap_from_end as cap_from_end @@ -44,9 +45,6 @@ from sqlbuild.compiler.planner.main.cursor_window.empty_cursor_window import empty_cursor_window from sqlbuild.compiler.planner.main.execution.future_cursor_warning import future_cursor_cap_warning from sqlbuild.compiler.planner.main.execution.maximum_start_warning import maximum_start_cap_warning -from sqlbuild.compiler.planner.main.execution.microbatch_limit import ( - microbatch_limit_warning, -) from sqlbuild.compiler.planner.models import ( CursorBounds, Duration, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py b/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py index 70b435d7e3..79b37bd434 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py @@ -4,10 +4,8 @@ from dataclasses import replace +from sqlbuild._native import replaces_incremental_relation from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists -from sqlbuild.compiler.planner.main.changes.relation_replacement import ( - replaces_incremental_relation, -) from sqlbuild.compiler.planner.main.cursor_window.empty_cursor_window import empty_cursor_window from sqlbuild.compiler.planner.main.execution.future_cursor_warning import ( future_cursor_cap_warning, diff --git a/src/sqlbuild/executor/run/_helpers/validation/contracts.py b/src/sqlbuild/executor/run/_helpers/validation/contracts.py index b9a2dcfc4c..0836cc0b4b 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/contracts.py +++ b/src/sqlbuild/executor/run/_helpers/validation/contracts.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import matching_dynamic_families from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.adapter.type_system.main.types_equal import types_equal @@ -13,7 +14,6 @@ RUNTIME_CONTRACT_MISSING_DECLARATIONS_CODE, RUNTIME_CONTRACT_TYPE_MISMATCH_CODE, ) -from sqlbuild.spec.contracts.main.matching_dynamic_families import matching_dynamic_families from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily diff --git a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py index e8503ef03f..e611499f86 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py +++ b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py @@ -22,9 +22,9 @@ from sqlbuild._native import cursor_render as render from sqlbuild._native import cursor_sentinel_from_token as sentinel_from_token from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token +from sqlbuild._native import resolve_cursor_intrinsics from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists -from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL from sqlbuild.compiler.planner.main.cursor_window.cursor_window_override_flags import ( cursor_window_override_flags, diff --git a/src/sqlbuild/executor/testing/main/resource_id.py b/src/sqlbuild/executor/testing/main/_resource_id.py similarity index 100% rename from src/sqlbuild/executor/testing/main/resource_id.py rename to src/sqlbuild/executor/testing/main/_resource_id.py diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py index 70ac8c252b..bd8cf49d2d 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py @@ -9,6 +9,7 @@ from typing import TextIO, cast from sqlbuild._native import ( + TransientStatusReporter, dbt_execution_ls_argv, parse_dbt_ls_json_lines, render_dbt_node_result, @@ -27,7 +28,6 @@ DbtNodeMessage, ) from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter _COMMAND_AND_SUBCOMMAND_ARGUMENT_COUNT: int = 2 diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py index 859ca77a78..44a93ea5ed 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py @@ -7,15 +7,16 @@ from sqlbuild._native import ( ConnectionProgressReporter, apply_command_warehouse, + build_effective_connection_config, render_dbt_execution_summary_footer, render_dbt_interop_plan, + resolve_target_config, ) from sqlbuild.cli.commands.main.dbt.dbt_sqlbuild_work import execute_dbt_sqlbuild_work from sqlbuild.cli.commands.main.execution.connection_progress import ( build_connection_progress_reporter, ) from sqlbuild.cli.commands.models import DbtSqlbuildWorkContext -from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.integrations.dbt._helpers.pipeline.execute import execute_dbt_commands @@ -36,7 +37,6 @@ ) from sqlbuild.presentation.models import DisplayOptions from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.models import ExecutionLimitsConfig diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py index 10ab3b2a7f..3532cc35e7 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py @@ -16,6 +16,7 @@ load_dbt_manifest_index, plan_dbt_interop_command, resolve_dbt_executable, + resolve_effective_adapter_name, route_dbt_interop_args, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -50,9 +51,6 @@ DbtManifestIndex, ) from sqlbuild.integrations.dbt.types import DbtInteropCommand -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) def resolve_dbt_execution_invocation( diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py index dd3a7ac299..6486e5cc97 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py @@ -10,10 +10,10 @@ from sqlbuild._native import ( apply_command_warehouse, + build_effective_connection_config, resolve_model_deferral_inputs, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.exceptions import PlannerInputError diff --git a/src/sqlbuild/integrations/dbt/_helpers/runtime/event_stream.py b/src/sqlbuild/integrations/dbt/_helpers/runtime/event_stream.py index adfd965658..754fc6c7ca 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/runtime/event_stream.py +++ b/src/sqlbuild/integrations/dbt/_helpers/runtime/event_stream.py @@ -14,11 +14,11 @@ from sqlbuild._native import ( DbtEventStream, DbtEventStreamUpdate, + TransientStatusReporter, format_dbt_active_node_status, ) from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError from sqlbuild.integrations.dbt.models import DbtNodeExecutionResult -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle _DBT_STATUS_REFRESH_SECONDS: float = 1.0 diff --git a/src/sqlbuild/integrations/dlt/_helpers/config.py b/src/sqlbuild/integrations/dlt/_helpers/config.py index 0b316bc06a..9027e67e0d 100644 --- a/src/sqlbuild/integrations/dlt/_helpers/config.py +++ b/src/sqlbuild/integrations/dlt/_helpers/config.py @@ -5,7 +5,7 @@ from dataclasses import replace from typing import cast -from sqlbuild.compiler.compile.main.expand_template_data import expand_template_data +from sqlbuild._native import expand_template_data from sqlbuild.integrations.dlt.models import DltResourceConfig, DltSourceConfig diff --git a/src/sqlbuild/integrations/ingestr/_helpers/command.py b/src/sqlbuild/integrations/ingestr/_helpers/command.py index 6e181b21c9..e9d0d733a7 100644 --- a/src/sqlbuild/integrations/ingestr/_helpers/command.py +++ b/src/sqlbuild/integrations/ingestr/_helpers/command.py @@ -6,7 +6,7 @@ from dataclasses import replace from typing import cast -from sqlbuild.compiler.compile.main.expand_template_data import expand_template_data +from sqlbuild._native import expand_template_data from sqlbuild.integrations.ingestr._helpers.destination import build_destination_uri from sqlbuild.integrations.ingestr.constants import INGESTR_INTEGRATION_KIND from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError diff --git a/src/sqlbuild/lint/_helpers/expansion.py b/src/sqlbuild/lint/_helpers/expansion.py index 08429e283c..b06a8229e5 100644 --- a/src/sqlbuild/lint/_helpers/expansion.py +++ b/src/sqlbuild/lint/_helpers/expansion.py @@ -7,22 +7,17 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import discover_project_inputs, external_identifiers_for_headers -from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter +from sqlbuild._native import ( + expand_sql_with_spans, + external_identifiers_for_headers, +) from sqlbuild.compiler.compile.constants import MACRO_TOKEN from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main.expand_sql_with_spans import expand_sql_with_spans -from sqlbuild.compiler.compile.main.sql_expansion_context import build_sql_expansion_context from sqlbuild.compiler.compile.models import ( CompiledSqlExpansion, - DeclarationScopeBuild, ExpansionSpan, SqlExpansionContext, ) -from sqlbuild.compiler.compile.types import TypedSqlValueRenderer -from sqlbuild.compiler.discovery.exceptions import DiscoveryError -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.lint._helpers.headers import lint_body_ranges, lint_file_role from sqlbuild.lint.classes.stop_context import LintStopContext @@ -43,9 +38,6 @@ PreparedLintFiles, ) from sqlbuild.lint.types import NativePreparedSql -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) _DEPENDENCY_INTRINSIC_PATTERN: re.Pattern[str] = re.compile( r"^__(?:ref|source)\s*\(", re.IGNORECASE @@ -55,57 +47,6 @@ ) -def build_lint_expansion_context( - *, - project_dir: Path, - value_renderer: TypedSqlValueRenderer | None = None, - discovered_inputs: DiscoveredProjectInputs | None = None, - declaration_scope: DeclarationScopeBuild | None = None, - static_declaration_scope: DeclarationScopeBuild | None = None, -) -> SqlExpansionContext: - """Build the expansion context, reporting compile failures as lint failures.""" - - try: - effective_discovered_inputs: DiscoveredProjectInputs = ( - discovered_inputs - if discovered_inputs is not None - else discover_project_inputs( - project_dir=project_dir, - sql_analysis_enabled_override=False, - extract_output_column_locations=False, - ) - ) - effective_renderer: TypedSqlValueRenderer = value_renderer or _resolve_value_renderer( - project_dir=project_dir, - discovered_inputs=effective_discovered_inputs, - ) - return build_sql_expansion_context( - project_dir=project_dir, - discovered_inputs=effective_discovered_inputs, - value_renderer=effective_renderer, - declaration_scope=declaration_scope, - static_declaration_scope=static_declaration_scope, - ) - except (AdapterUserError, CompileInputError, DiscoveryError) as error: - raise ProjectCompileError( - "compiler-integrated Rules check the SQL your project actually produces, " - "so the project must " - f"compile first: {error}" - ) from error - - -def _resolve_value_renderer( - *, project_dir: Path, discovered_inputs: DiscoveredProjectInputs -) -> TypedSqlValueRenderer: - return resolve_adapter( - adapter_name=resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ), - project_dir=project_dir, - ) - - def prepare_lint_body( *, role: LintFileRole, diff --git a/src/sqlbuild/lint/_helpers/fixes.py b/src/sqlbuild/lint/_helpers/fixes.py index 39c8893de8..0dbc1b6233 100644 --- a/src/sqlbuild/lint/_helpers/fixes.py +++ b/src/sqlbuild/lint/_helpers/fixes.py @@ -2,13 +2,11 @@ from __future__ import annotations -import os -import tempfile from collections import Counter from dataclasses import dataclass, field, replace from pathlib import Path -from sqlbuild._native import build_compiled_project, run_lint +from sqlbuild._native import build_compiled_project, compiled_relation_keys, run_lint from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView, TypedSqlValueRenderer @@ -21,7 +19,6 @@ ) from sqlbuild.lint.exceptions import ProjectCompileError from sqlbuild.lint.main.compile_facts import compile_facts -from sqlbuild.lint.main.compiled_relation_keys import compiled_relation_keys from sqlbuild.lint.main.fix_verdict import fix_verdict from sqlbuild.lint.models import ( CompileFacts, @@ -292,20 +289,6 @@ def _decline( ) -def write_atomically(*, path: Path, contents: str) -> None: - descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) - temporary_path: Path = Path(temporary_name) - try: - os.fchmod(descriptor, path.stat().st_mode) - with os.fdopen(descriptor, "w", encoding="utf-8", newline="") as handle: - _ = handle.write(contents) - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary_path, path) - finally: - temporary_path.unlink(missing_ok=True) - - def plan_rule_fixes( *, project_dir: Path, diff --git a/src/sqlbuild/lint/main/build_expansion_context.py b/src/sqlbuild/lint/main/build_expansion_context.py deleted file mode 100644 index 3a3cefe84f..0000000000 --- a/src/sqlbuild/lint/main/build_expansion_context.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Prepare invocation-local SQL expansion evidence for compiler-integrated lint.""" - -from pathlib import Path - -from sqlbuild.compiler.compile.models import DeclarationScopeBuild, SqlExpansionContext -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.lint._helpers.expansion import build_lint_expansion_context - - -def build_expansion_context( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - declaration_scope: DeclarationScopeBuild | None = None, - static_declaration_scope: DeclarationScopeBuild | None = None, -) -> SqlExpansionContext: - return build_lint_expansion_context( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - declaration_scope=declaration_scope, - static_declaration_scope=static_declaration_scope, - ) diff --git a/src/sqlbuild/lint/main/compiled_relation_keys.py b/src/sqlbuild/lint/main/compiled_relation_keys.py deleted file mode 100644 index de49299728..0000000000 --- a/src/sqlbuild/lint/main/compiled_relation_keys.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Declared unique keys of a compiled project's relations.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.lint.main.declared_relation_keys import declared_relation_keys -from sqlbuild.lint.types import RelationKeys - - -def compiled_relation_keys(project: CompiledProjectView) -> RelationKeys: - """Collect declared keys of compiled models, seeds and sources.""" - - return declared_relation_keys( - model_rows=(model.relation_key_row for model in project.models), - seeds=(seed.schema_entry for seed in project.seeds), - sources=(source.source_entry for source in project.sources), - ) diff --git a/src/sqlbuild/lint/main/declared_relation_keys.py b/src/sqlbuild/lint/main/declared_relation_keys.py deleted file mode 100644 index 37237157d4..0000000000 --- a/src/sqlbuild/lint/main/declared_relation_keys.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Declared unique keys of project relations, for ranking determinism proofs.""" - -from __future__ import annotations - -from collections.abc import Iterable - -import sqlbuild._native as _native -from sqlbuild.lint.constants import RELATION_KIND_SEED, RELATION_KIND_SOURCE -from sqlbuild.lint.types import RelationKeyRow, RelationKeys -from sqlbuild.spec.contracts.models import SchemaSeedEntry, SourceEntry - - -def declared_relation_keys( - *, - model_rows: Iterable[RelationKeyRow], - seeds: Iterable[SchemaSeedEntry], - sources: Iterable[SourceEntry], -) -> RelationKeys: - """Keys that prove one row per key, from model key rows and seed and source entries.""" - - rows: list[RelationKeyRow] = [ - *model_rows, - *(_native.relation_key_row(RELATION_KIND_SEED, seed.name, {}, seed) for seed in seeds), - *( - _native.relation_key_row(RELATION_KIND_SOURCE, source.name, {}, source) - for source in sources - ), - ] - keys: RelationKeys = {} - for identity, found in _native.declared_relation_keys(rows).items(): - keys[identity] = tuple(map(tuple, found)) - return keys diff --git a/src/sqlbuild/lint/main/write_atomically.py b/src/sqlbuild/lint/main/write_atomically.py deleted file mode 100644 index 96599a0f29..0000000000 --- a/src/sqlbuild/lint/main/write_atomically.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public atomic source-file writer.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.lint._helpers.fixes import write_atomically as _write_atomically - - -def write_atomically(*, path: Path, contents: str) -> None: - """Replace a text file atomically.""" - - _write_atomically(path=path, contents=contents) diff --git a/src/sqlbuild/microbatches/_helpers/sql.py b/src/sqlbuild/microbatches/_helpers/sql.py index cf549ba41a..7f8928e42c 100644 --- a/src/sqlbuild/microbatches/_helpers/sql.py +++ b/src/sqlbuild/microbatches/_helpers/sql.py @@ -4,6 +4,7 @@ from collections.abc import Callable +from sqlbuild._native import render_state_sql_literal from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec from sqlbuild.microbatches.constants import ( @@ -14,7 +15,6 @@ ) from sqlbuild.microbatches.exceptions import MicrobatchStateError from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope -from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal from sqlbuild.sql_values.types import StateSqlValueType diff --git a/src/sqlbuild/presentation/classes/transient_status_reporter.py b/src/sqlbuild/presentation/classes/transient_status_reporter.py deleted file mode 100644 index 9a3aa9e5e9..0000000000 --- a/src/sqlbuild/presentation/classes/transient_status_reporter.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Transient status reporter class.""" - -from __future__ import annotations - -import atexit -from contextlib import AbstractContextManager -from typing import TYPE_CHECKING, TextIO - -from sqlbuild.errors.contracts.exceptions import SharedInputError -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator -from sqlbuild.presentation.main._progress_spinners_disabled import progress_spinners_disabled -from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator - -if TYPE_CHECKING: - from rich.console import Console - from rich.status import Status - - -class TransientStatusReporter: - """Render one transient status line with optional spinner updates.""" - - def __init__( - self, - *, - stream: TextIO, - use_color: bool = False, - enabled: bool = True, - ) -> None: - self._stream: TextIO = stream - self._lines: TransientLineCoordinator = shared_transient_line_coordinator() - self._style: CliStyle = CliStyle(use_color=use_color) - self._enabled: bool = ( - enabled - and not progress_spinners_disabled() - and hasattr(stream, "isatty") - and stream.isatty() - ) - self._console: Console | None = None - if self._enabled: - from rich.console import Console - - self._console = Console(file=stream, no_color=(not use_color)) - self._status_context: AbstractContextManager[Status] | None = None - self._status: Status | None = None - self._active: bool = False - self._message: str | None = None - atexit.register(self.close) - - def start(self, message: str) -> None: - if self._enabled: - with self._lines.lock: - self.close() - self._message = message - self._enter_status() - self._lines.claim(stream=self._stream, owner=self) - self._active = True - return - self._active = True - self._write_message(message=message, dim_output=True) - - def update(self, message: str) -> None: - if self._enabled: - with self._lines.lock: - if self._message is not None: - self._message = message - if self._status is not None: - self._status.update(message) - return - self._write_message(message=message, dim_output=True) - - def complete(self, *, message: str, blank_line_after: bool = False) -> None: - self.close() - self._write_message(message=message, dim_output=True) - if blank_line_after: - self.write_blank_line() - - def complete_styled(self, *, message: str, blank_line_after: bool = False) -> None: - """Complete with a pre-styled message written verbatim.""" - - self.close() - self._lines.write_persistent( - stream=self._stream, text=f"{message}\n" + ("\n" if blank_line_after else "") - ) - - def error(self, message: str) -> None: - self.close() - self._write_message(message=message, dim_output=False) - - def close(self) -> None: - with self._lines.lock: - self._lines.release(owner=self) - self._exit_status() - self._message = None - self._active = False - - def clear_transient_line(self) -> None: - """Erase the live status so a persistent line can be written in its place.""" - - self._exit_status() - - def redraw_transient_line(self) -> None: - """Restart the live status below persistent output written while it was active.""" - - if self._message is not None and self._status_context is None: - self._enter_status() - - def report_preflight_progress(self, message: str) -> None: - """Render one preflight progress update.""" - - if message.startswith("Prepared "): - self.complete(message=message, blank_line_after=True) - return - if not self._active: - self.start(message) - return - self.update(message) - - def write_blank_line(self) -> None: - self._lines.write_persistent(stream=self._stream, text="\n") - - def _enter_status(self) -> None: - if self._console is None or self._message is None: - raise SharedInputError( - "status reporter was enabled without an initialized console", - code="G001", - ) - self._status_context = self._console.status(status=self._message, spinner="dots") - self._status = self._status_context.__enter__() - - def _exit_status(self) -> None: - if self._status_context is not None: - self._status_context.__exit__(None, None, None) - self._status_context = None - self._status = None - - def _write_message(self, *, message: str, dim_output: bool) -> None: - formatted_message: str = self._style.muted(message) if dim_output else message - self._lines.write_persistent(stream=self._stream, text=f"{formatted_message}\n") diff --git a/src/sqlbuild/presentation/main/_maybe_status.py b/src/sqlbuild/presentation/main/_maybe_status.py index 3e12846418..d9cb7214cd 100644 --- a/src/sqlbuild/presentation/main/_maybe_status.py +++ b/src/sqlbuild/presentation/main/_maybe_status.py @@ -6,8 +6,8 @@ from collections.abc import Iterator from contextlib import contextmanager +from sqlbuild._native import TransientStatusReporter from sqlbuild.presentation._helpers.progress import progress_spinners_disabled -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter @contextmanager diff --git a/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py b/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py index 852339009f..512d3f411e 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py @@ -10,12 +10,12 @@ from pathlib import Path from types import ModuleType -from sqlbuild._native import builtin_rules +from sqlbuild._native import builtin_rules, selected_rule_codes +from sqlbuild.rule_engine._helpers.engine.custom_payloads import custom_rule_payloads from sqlbuild.rule_engine._helpers.engine.definition import ( resolve_rule_signature, rule_from_value, ) -from sqlbuild.rule_engine._helpers.engine.native import native_selected_codes from sqlbuild.rule_engine.constants import ( INIT_MODULE_NAME, RULE_DECORATOR_TOKEN, @@ -77,11 +77,13 @@ def select_rules( """Resolve the active catalogue through the native Fensu rules owner.""" by_code: dict[str, Rule] = {rule.code: rule for rule in catalogue} - codes: tuple[str, ...] = native_selected_codes( + codes: list[str] = selected_rule_codes( config=config, - catalogue=catalogue, - project_dir=project_dir, - custom_payloads=custom_payloads, + custom_payloads=( + custom_rule_payloads(catalogue=catalogue, project_dir=project_dir) + if custom_payloads is None + else custom_payloads + ), ) try: return tuple(by_code[code] for code in codes) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py index 4cdbe8af63..c017406120 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py @@ -8,18 +8,15 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import rule_selected, selected_rule_codes from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity +from sqlbuild.rule_engine._helpers.engine.custom_payloads import custom_rule_payloads from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached -from sqlbuild.rule_engine._helpers.engine.native import ( - custom_rule_payloads, - finalize_native_findings, - rule_selected, - selected_codes, -) from sqlbuild.rule_engine._helpers.run.native_rows import ( build_rules_request, evaluate_rules_request, + finalize_rules_rows, ) from sqlbuild.rule_engine.constants import TYPE_PROOF_RULE_CODES from sqlbuild.rule_engine.models import ( @@ -113,7 +110,7 @@ def _evaluate_native_rows( *custom.findings, ) if not defer_suppressions: - findings = finalize_native_findings( + findings = finalize_rules_rows( config=config, project_dir=project_dir, evaluated_codes=native.selected_codes, @@ -175,6 +172,6 @@ def _selected_custom_rules( if not custom_payloads: return () selected: frozenset[str] = frozenset( - selected_codes(config=config, custom_payloads=custom_payloads) + selected_rule_codes(config=config, custom_payloads=custom_payloads) ) return tuple(rule for rule in catalogue if rule.custom and rule.code in selected) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_payloads.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_payloads.py new file mode 100644 index 0000000000..d18c5786e7 --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_payloads.py @@ -0,0 +1,83 @@ +"""Describe catalogued custom rules, reading their user functions, for native selection.""" + +from __future__ import annotations + +import inspect +from pathlib import Path + +from sqlbuild._native import ( + custom_rule_implementation_fingerprint, + custom_rule_import_closure, + custom_rule_test_evidence, +) +from sqlbuild.rule_engine.models import Rule + + +def custom_rule_payloads( + *, catalogue: tuple[Rule, ...], project_dir: Path +) -> list[dict[str, object]]: + """Describe every catalogued custom rule for native selection and evaluation.""" + + closures: dict[str, tuple[Path, ...]] = {} + return [ + _custom_rule_payload( + rule=rule, + project_dir=project_dir, + import_closure=_shared_import_closure( + rule=rule, project_dir=project_dir, cache=closures + ), + ) + for rule in catalogue + if rule.custom + ] + + +def _custom_rule_payload( + *, rule: Rule, project_dir: Path, import_closure: tuple[Path, ...] +) -> dict[str, object]: + return { + "code": rule.code, + "family": rule.family, + "slug": rule.slug, + "message": rule.message, + "remediation": rule.remediation, + "enabled_by_default": rule.enabled_by_default, + "implementation_fingerprint": custom_rule_implementation_fingerprint( + rule=rule, project_dir=project_dir, import_closure=import_closure + ), + **_custom_rule_source_payload(rule=rule, project_dir=project_dir), + "project_wide": rule.project_wide, + "check_name": getattr(rule.check, "__name__", ""), + "test_case_count": len(custom_rule_test_evidence(rule=rule, project_dir=project_dir)), + } + + +def _shared_import_closure( + *, rule: Rule, project_dir: Path, cache: dict[str, tuple[Path, ...]] +) -> tuple[Path, ...]: + if rule.source is None: + return () + source: str = str(Path(rule.source).resolve()) + if source not in cache: + cache[source] = custom_rule_import_closure(rule=rule, project_dir=project_dir) + return cache[source] + + +def _custom_rule_source_payload(*, rule: Rule, project_dir: Path) -> dict[str, object]: + source: str | None = rule.source + if source is not None: + source_path: Path = Path(source).resolve() + root: Path = project_dir.resolve() + if source_path.is_relative_to(root): + source = source_path.relative_to(root).as_posix() + try: + source_line: int = inspect.getsourcelines(rule.check)[1] + except (OSError, TypeError): + source_line = 1 + check_owner: str = getattr(rule.check, "__qualname__", getattr(rule.check, "__name__", "")) + return { + "source": source, + "source_line": source_line, + "source_column": 1, + "owner": f"{source or 'rules'}:{check_owner}", + } diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py deleted file mode 100644 index b1c6147abe..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Typed Python adapter for the private native rules engine.""" - -from __future__ import annotations - -import inspect -import json -from dataclasses import asdict -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild._native import ( - custom_rule_implementation_fingerprint, - custom_rule_import_closure, - custom_rule_test_evidence, -) -from sqlbuild.rule_engine._helpers.run.native_rows import ( - finalize_rules_rows, -) -from sqlbuild.rule_engine.constants import ( - RULES_NATIVE_API_VERSION, -) -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import ( - Finding, - Rule, - RulesConfig, -) - - -def finalize_native_findings( - *, - config: RulesConfig, - project_dir: Path, - evaluated_codes: tuple[str, ...], - findings: tuple[Finding, ...], -) -> tuple[Finding, ...]: - """Apply exception policy once to completed SQL, native, and custom findings.""" - - return finalize_rules_rows( - config=config, - project_dir=project_dir, - evaluated_codes=evaluated_codes, - findings=findings, - ) - - -def native_selected_codes( - *, - config: RulesConfig, - catalogue: tuple[Rule, ...], - project_dir: Path, - custom_payloads: list[dict[str, object]] | None = None, -) -> tuple[str, ...]: - """Resolve the active ruleset through the native Fensu adapter.""" - - return selected_codes( - config=config, - custom_payloads=( - custom_rule_payloads(catalogue=catalogue, project_dir=project_dir) - if custom_payloads is None - else custom_payloads - ), - ) - - -def selected_codes( - *, config: RulesConfig, custom_payloads: list[dict[str, object]] -) -> tuple[str, ...]: - request: dict[str, object] = { - "version": RULES_NATIVE_API_VERSION, - "config": _config_payload(config), - "custom_rules": custom_payloads, - } - try: - payload: object = json.loads( - _native.selected_codes_json(json.dumps(request, sort_keys=True, default=str)) - ) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - if not isinstance(payload, list) or any(not isinstance(code, str) for code in payload): - raise RulesError("native rules engine returned invalid selected rule codes") - return tuple(payload) - - -def render_native_owned_skill(*, content: str, input_fingerprint: str) -> str: - """Attach Fensu schema-v2 ownership to generated guidance.""" - - try: - return _native.render_owned_skill(content, input_fingerprint) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - - -def native_skill_freshness(*, content: str | None, input_fingerprint: str) -> str: - """Classify generated guidance through the Fensu ownership contract.""" - - return _native.skill_freshness(content, input_fingerprint) - - -def _config_payload(config: RulesConfig) -> dict[str, object]: - return asdict(config) - - -def rule_selected(*, config: RulesConfig, code: str) -> bool: - selected: bool = any(code.startswith(selector) for selector in config.select) - ignored: bool = any(code.startswith(selector) for selector in config.ignore) - return selected and not ignored - - -def custom_rule_payloads( - *, catalogue: tuple[Rule, ...], project_dir: Path -) -> list[dict[str, object]]: - """Describe every catalogued custom rule for native selection and evaluation.""" - - closures: dict[str, tuple[Path, ...]] = {} - return [ - _custom_rule_payload( - rule=rule, - project_dir=project_dir, - import_closure=_shared_import_closure( - rule=rule, project_dir=project_dir, cache=closures - ), - ) - for rule in catalogue - if rule.custom - ] - - -def _custom_rule_payload( - *, rule: Rule, project_dir: Path, import_closure: tuple[Path, ...] -) -> dict[str, object]: - return { - "code": rule.code, - "family": rule.family, - "slug": rule.slug, - "message": rule.message, - "remediation": rule.remediation, - "enabled_by_default": rule.enabled_by_default, - "implementation_fingerprint": custom_rule_implementation_fingerprint( - rule=rule, project_dir=project_dir, import_closure=import_closure - ), - **_custom_rule_source_payload(rule=rule, project_dir=project_dir), - "project_wide": rule.project_wide, - "check_name": getattr(rule.check, "__name__", ""), - "test_case_count": len(custom_rule_test_evidence(rule=rule, project_dir=project_dir)), - } - - -def _shared_import_closure( - *, rule: Rule, project_dir: Path, cache: dict[str, tuple[Path, ...]] -) -> tuple[Path, ...]: - if rule.source is None: - return () - source: str = str(Path(rule.source).resolve()) - if source not in cache: - cache[source] = custom_rule_import_closure(rule=rule, project_dir=project_dir) - return cache[source] - - -def _custom_rule_source_payload(*, rule: Rule, project_dir: Path) -> dict[str, object]: - source: str | None = rule.source - if source is not None: - source_path: Path = Path(source).resolve() - root: Path = project_dir.resolve() - if source_path.is_relative_to(root): - source = source_path.relative_to(root).as_posix() - try: - source_line: int = inspect.getsourcelines(rule.check)[1] - except (OSError, TypeError): - source_line = 1 - check_owner: str = getattr(rule.check, "__qualname__", getattr(rule.check, "__name__", "")) - return { - "source": source, - "source_line": source_line, - "source_column": 1, - "owner": f"{source or 'rules'}:{check_owner}", - } diff --git a/src/sqlbuild/rule_engine/_helpers/guidance/skill_installation.py b/src/sqlbuild/rule_engine/_helpers/guidance/skill_installation.py index 83cedacfeb..cbe6fa7817 100644 --- a/src/sqlbuild/rule_engine/_helpers/guidance/skill_installation.py +++ b/src/sqlbuild/rule_engine/_helpers/guidance/skill_installation.py @@ -7,7 +7,7 @@ from stat import S_IMODE from tempfile import NamedTemporaryFile -from sqlbuild.rule_engine._helpers.engine.native import native_skill_freshness +from sqlbuild._native import skill_freshness from sqlbuild.rule_engine.constants import REPLACEABLE_SKILL_STATES from sqlbuild.rule_engine.exceptions import RulesError @@ -18,7 +18,7 @@ def install_rendered_skills( """Validate and atomically replace all generated skill targets.""" freshness: tuple[str, ...] = tuple( - native_skill_freshness( + skill_freshness( content=path.read_text(encoding="utf-8") if path.is_file() else None, input_fingerprint=input_fingerprint, ) diff --git a/src/sqlbuild/rule_engine/_helpers/guidance/skills.py b/src/sqlbuild/rule_engine/_helpers/guidance/skills.py index 10a496c931..266b9ee7f3 100644 --- a/src/sqlbuild/rule_engine/_helpers/guidance/skills.py +++ b/src/sqlbuild/rule_engine/_helpers/guidance/skills.py @@ -5,7 +5,7 @@ from collections.abc import Mapping from pathlib import Path -from sqlbuild.rule_engine._helpers.engine.native import render_native_owned_skill +from sqlbuild._native import render_owned_skill from sqlbuild.rule_engine._helpers.engine.ruleset import resolve_ruleset from sqlbuild.rule_engine._helpers.guidance.thresholds import format_threshold_lines from sqlbuild.rule_engine.constants import ( @@ -104,7 +104,7 @@ def render_skills(*, config: RulesConfig, project_dir: Path) -> tuple[str, str]: body.append(f"- Replace retired `{retired}` with `{replacement}`") content: str = "\n".join(body).rstrip() + "\n" return ( - render_native_owned_skill(content=content, input_fingerprint=ruleset.fingerprint), + render_owned_skill(content=content, input_fingerprint=ruleset.fingerprint), ruleset.fingerprint, ) diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py index 843d59146f..7af6ef3860 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py +++ b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py @@ -9,11 +9,10 @@ import orjson import sqlbuild._native as _native -from sqlbuild._native import compiled_project_facts, declaration_rows +from sqlbuild._native import compiled_project_facts, declaration_rows, scope_metadata_projection from sqlbuild.adapter.type_system.main.types_equal import types_equal from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity -from sqlbuild.compiler.scopes.main.scope_metadata import scope_metadata_projection from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine.constants import ( NATIVE_RULES_CACHE_FILE, diff --git a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py index 9e65027fea..962a16c4ad 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py @@ -7,7 +7,7 @@ from pathlib import Path from threading import Event -from sqlbuild._native import run_lint +from sqlbuild._native import build_lint_expansion_context, run_lint from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledSqlExpansion, @@ -18,7 +18,6 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.lint.classes.stop_context import LintStopContext -from sqlbuild.lint.main.build_expansion_context import build_expansion_context from sqlbuild.lint.models import LintConfig, LintRunResult from sqlbuild.rule_engine._helpers.engine.custom_dispatch import evaluate_native, start_custom_rules from sqlbuild.rule_engine._helpers.run.rules import ( @@ -135,7 +134,7 @@ def _prepare_sql_lint( ) -> PreparedSqlLintResult: stop_token: Token[Event | None] = LintStopContext.active.set(stop) try: - context: SqlExpansionContext = build_expansion_context( + context: SqlExpansionContext = build_lint_expansion_context( project_dir=project_dir, discovered_inputs=discovered_inputs, static_declaration_scope=static_declaration_scope, diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index e4bed6d647..ccbf7171c9 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -8,7 +8,10 @@ from pathlib import Path from sqlbuild._native import ( + build_lint_expansion_context, collect_lint_project_files, + compiled_relation_keys, + declared_relation_keys, encode_sql_rule_findings, load_sql_rule_cache, lookup_sql_rule_cache, @@ -44,21 +47,16 @@ SAFE_RULE_FIX_CODES, ) from sqlbuild.lint.exceptions import NativeLintError -from sqlbuild.lint.main.build_expansion_context import build_expansion_context -from sqlbuild.lint.main.compiled_relation_keys import compiled_relation_keys -from sqlbuild.lint.main.declared_relation_keys import declared_relation_keys from sqlbuild.lint.models import LintConfig, LintRunResult, LintViolation from sqlbuild.lint.types import RelationKeys from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules from sqlbuild.rule_engine._helpers.engine.config import resolve_rule_ignore_selectors +from sqlbuild.rule_engine._helpers.engine.custom_payloads import custom_rule_payloads from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules -from sqlbuild.rule_engine._helpers.engine.native import ( - custom_rule_payloads, - finalize_native_findings, -) from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine._helpers.run.findings import group_unevaluated_findings from sqlbuild.rule_engine._helpers.run.memory import release_freed_memory +from sqlbuild.rule_engine._helpers.run.native_rows import finalize_rules_rows from sqlbuild.rule_engine.constants import SQL_RULES_CACHE_FILE, TYPE_PROOF_RULE_CODES from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main.load_config import load_rules_config @@ -197,7 +195,7 @@ def finish_rule_run( ) -> RulesRunResult: """Finalize the combined SQL, built-in and custom findings of one run.""" - finalized: tuple[Finding, ...] = finalize_native_findings( + finalized: tuple[Finding, ...] = finalize_rules_rows( config=plan.config, project_dir=plan.project_dir, evaluated_codes=tuple(rule.code for rule in plan.selected), @@ -577,7 +575,7 @@ def _lint_expansion_context( if expansion_reuse is not None and expansion_reuse.discovered_inputs is discovered_inputs else None ) - return build_expansion_context( + return build_lint_expansion_context( project_dir=project_dir, discovered_inputs=discovered_inputs, declaration_scope=declaration_scope, diff --git a/src/sqlbuild/rule_engine/_helpers/sql/ast.py b/src/sqlbuild/rule_engine/_helpers/sql/ast.py deleted file mode 100644 index a24a0b4ff4..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/sql/ast.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Polyglot AST traversal and payload helpers for rules.""" - -from __future__ import annotations - -from collections.abc import Iterator -from typing import Any, cast - -from sqlbuild.rule_engine.constants import AST_SELECT_KIND, NAMED_CTE_TUPLE_SIZE - - -def payload(node: Any) -> dict[str, object]: - """Return the single expression payload from a Polyglot node.""" - - raw: object = node.to_dict() - if not isinstance(raw, dict) or len(raw) != 1: - return {} - value: object = next(iter(raw.values())) - return cast(dict[str, object], value) if isinstance(value, dict) else {} - - -def kind(node: Any) -> str: - """Return a stable lowercase Polyglot node kind.""" - - return str(getattr(node, "key", "") or getattr(node, "kind", "")).lower() - - -def nodes(*, root: Any, wanted: str) -> Iterator[Any]: - """Yield every node whose stable kind matches wanted.""" - - for node in root.walk(): - if kind(node) == wanted: - yield node - - -def name_value(value: object) -> str: - """Extract a name from Polyglot's nested name payload.""" - - if isinstance(value, str): - return value - if isinstance(value, dict): - mapping: dict[str, object] = cast(dict[str, object], value) - raw: object = mapping.get("name") - if isinstance(raw, str): - return raw - if isinstance(raw, dict): - return name_value(raw) - return "" - - -def cte_name(node: Any) -> str: - if isinstance(node, tuple) and len(node) == NAMED_CTE_TUPLE_SIZE: - return str(node[0]) - return str(getattr(node, "alias_or_name", "") or getattr(node, "name", "")) - - -def top_level_ctes(root: Any) -> tuple[Any, ...]: - """Return top-level CTE nodes in authored order.""" - - data: dict[str, object] = payload(root) - with_payload: object = data.get("with") - raw_ctes: object = ( - cast(dict[str, object], with_payload).get("ctes") - if isinstance(with_payload, dict) - else None - ) - if not isinstance(raw_ctes, list): - return () - candidates: tuple[Any, ...] = tuple(root.walk()) - matched_ids: set[int] = set() - ctes: list[tuple[str, Any]] = [] - for raw_cte in raw_ctes: - if not isinstance(raw_cte, dict): - return () - cte_mapping: dict[str, object] = cast(dict[str, object], raw_cte) - alias: str = name_value(cte_mapping.get("alias")) - raw_body: object = cte_mapping.get("this") - body: Any | None = None - for candidate in candidates: - if id(candidate) in matched_ids or candidate is root: - continue - if candidate.to_dict() == raw_body: - body = candidate - matched_ids.add(id(candidate)) - break - if not alias or body is None: - return () - ctes.append((alias, body)) - return tuple(ctes) - - -def select_payload(node: Any) -> dict[str, object]: - return payload(node) if kind(node) == AST_SELECT_KIND else {} diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index d985125218..b388d81a9b 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -76,7 +76,6 @@ SKILL_STALE: str = "stale" REPLACEABLE_SKILL_STATES: frozenset[str] = frozenset({SKILL_FRESH, SKILL_MISSING, SKILL_STALE}) -AST_SELECT_KIND: str = "select" AST_CTE_KIND: str = "cte" CONTRACT_ENFORCED: str = "enforced" TYPE_PROOF_RULE_CODES: frozenset[str] = frozenset({"SQBRCONTRACT105"}) @@ -86,7 +85,6 @@ TIMESTAMP_TYPE: str = "TIMESTAMP" -NAMED_CTE_TUPLE_SIZE: int = 2 TARGET_DIRECTORY_NAME: str = "target" RULE_DECORATOR_TOKEN: str = "@rule" INIT_MODULE_NAME: str = "__init__" diff --git a/src/sqlbuild/rule_engine/main/skills.py b/src/sqlbuild/rule_engine/main/skills.py index e9aed9052d..68e6c5846e 100644 --- a/src/sqlbuild/rule_engine/main/skills.py +++ b/src/sqlbuild/rule_engine/main/skills.py @@ -9,7 +9,7 @@ from filelock import FileLock -from sqlbuild.rule_engine._helpers.engine.native import native_skill_freshness +from sqlbuild._native import skill_freshness from sqlbuild.rule_engine._helpers.guidance.skill_installation import install_rendered_skills from sqlbuild.rule_engine._helpers.guidance.skills import render_skills from sqlbuild.rule_engine.constants import ( @@ -28,7 +28,7 @@ def install_skills(*, config: RulesConfig, project_dir: Path, check: bool) -> bo targets: tuple[Path, ...] = tuple(project_dir / value for value in RULES_SKILL_PATHS) if check: return all( - native_skill_freshness( + skill_freshness( content=path.read_text(encoding="utf-8") if path.is_file() else None, input_fingerprint=input_fingerprint, ) diff --git a/src/sqlbuild/spec/contracts/_helpers/dynamic_columns.py b/src/sqlbuild/spec/contracts/_helpers/dynamic_columns.py deleted file mode 100644 index 0a96e4bd31..0000000000 --- a/src/sqlbuild/spec/contracts/_helpers/dynamic_columns.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Shared matching helpers for dynamic column contract families.""" - -from __future__ import annotations - -from functools import lru_cache - -import sqlbuild._native as _native -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -def dynamic_family_accepts_name(*, family: SchemaDynamicColumnFamily, column_name: str) -> bool: - """Return whether a runtime column name belongs to a declared family.""" - - if family.name_pattern is None: - return True - return _compiled_name_pattern(family.name_pattern).matches(column_name) - - -def matching_dynamic_families( - *, families: tuple[SchemaDynamicColumnFamily, ...], column_name: str -) -> tuple[SchemaDynamicColumnFamily, ...]: - """Return every family whose optional naming constraint accepts a column.""" - - return tuple( - family - for family in families - if dynamic_family_accepts_name(family=family, column_name=column_name) - ) - - -@lru_cache(maxsize=256) -def _compiled_name_pattern(pattern: str) -> _native.NamePattern: - return _native.NamePattern(pattern) diff --git a/src/sqlbuild/spec/contracts/_helpers/project_config.py b/src/sqlbuild/spec/contracts/_helpers/project_config.py deleted file mode 100644 index bd620b96f9..0000000000 --- a/src/sqlbuild/spec/contracts/_helpers/project_config.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Project configuration resolution implementations.""" - -from __future__ import annotations - -from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - ScenarioConfig, - ScenarioSnapshotLimitsConfig, -) -from sqlbuild.sql_values.types import CollectionRendering - - -def resolve_effective_collection_rendering( - *, - project_config: ProjectConfig, - declaration_override: CollectionRendering | None, -) -> CollectionRendering: - """Resolve declaration rendering over project configuration.""" - - if declaration_override is not None: - return declaration_override - return project_config.constants.collection_rendering - - -def resolve_effective_adapter_name( - *, project_config: ProjectConfig, local_config: LocalConfig -) -> str: - """Resolve the effective adapter name, allowing local override.""" - - if local_config.adapter is not None: - return local_config.adapter - return project_config.adapter - - -def resolve_effective_scenario_config( - *, project_config: ProjectConfig, local_config: LocalConfig -) -> ScenarioConfig: - """Resolve scenario config, allowing local overrides to replace project rules.""" - - local_type_overrides: dict[str, dict[str, str]] = { - dialect: dict(rules) - for dialect, rules in project_config.scenario.local_type_overrides.items() - } - dialect: str - rules: dict[str, str] - for dialect, rules in local_config.scenario.local_type_overrides.items(): - local_type_overrides.setdefault(dialect, {}).update(rules) - project_limits: ScenarioSnapshotLimitsConfig = project_config.scenario.snapshot_limits - local_limits: ScenarioSnapshotLimitsConfig = local_config.scenario.snapshot_limits - return ScenarioConfig( - run_namespace=( - local_config.scenario.run_namespace - if local_config.scenario.run_namespace is not None - else project_config.scenario.run_namespace - ), - local_type_overrides=local_type_overrides, - snapshot_limits=ScenarioSnapshotLimitsConfig( - max_rows_per_relation=_resolve_optional_int_override( - project_value=project_limits.max_rows_per_relation, - local_value=local_limits.max_rows_per_relation, - ), - max_total_rows=_resolve_optional_int_override( - project_value=project_limits.max_total_rows, - local_value=local_limits.max_total_rows, - ), - max_bytes_per_relation=_resolve_optional_int_override( - project_value=project_limits.max_bytes_per_relation, - local_value=local_limits.max_bytes_per_relation, - ), - max_total_bytes=_resolve_optional_int_override( - project_value=project_limits.max_total_bytes, - local_value=local_limits.max_total_bytes, - ), - ), - ) - - -def scenario_local_type_overrides_for_dialect( - *, scenario_config: ScenarioConfig, sql_analysis_dialect: str | None -) -> dict[str, str]: - """Return global and dialect-specific scenario local type override rules.""" - - overrides: dict[str, str] = dict(scenario_config.local_type_overrides.get("*", {})) - if sql_analysis_dialect is not None: - overrides.update(scenario_config.local_type_overrides.get(sql_analysis_dialect, {})) - return overrides - - -def _resolve_optional_int_override( - *, project_value: int | None, local_value: int | None -) -> int | None: - return local_value if local_value is not None else project_value diff --git a/src/sqlbuild/spec/contracts/_helpers/targets.py b/src/sqlbuild/spec/contracts/_helpers/targets.py deleted file mode 100644 index f35d453908..0000000000 --- a/src/sqlbuild/spec/contracts/_helpers/targets.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Target selection and target merging, resolved natively over the loaded config objects.""" - -from __future__ import annotations - -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.models import ( - ClonePolicy, - ExecutionLimitsConfig, - LocalConfig, - ProjectConfig, - TargetConfig, - TargetDiffConfig, - TargetWarehousesConfig, -) - -_GROUPS: dict[str, type[object]] = { - "clone": ClonePolicy, - "execution_limits": ExecutionLimitsConfig, - "diff": TargetDiffConfig, - "warehouses": TargetWarehousesConfig, -} - - -def resolve_target_name( - *, - project_config: ProjectConfig, - local_config: LocalConfig, - selected_target: str | None, -) -> str | None: - """Resolve the effective target name.""" - - name, error = _native.effective_config_target_name( - project_config, local_config, selected_target - ) - if error is not None: - raise SpecConfigError(error) - return name - - -def resolve_target_config( - *, - project_config: ProjectConfig, - local_config: LocalConfig, - target_name: str, -) -> TargetConfig: - """Merge project target config with local developer overrides.""" - - rows, error = _native.effective_config_target_fields( - (project_config, local_config), target_name, TargetConfig() - ) - if rows is None: - raise SpecConfigError(error or "") - values: dict[str, Any] = {} - grouped: dict[str, dict[str, Any]] = {group: {} for group in _GROUPS} - for group, name, value in rows: - (values if group is None else grouped[group])[name] = value - nested: dict[str, Any] = {group: kind(**grouped[group]) for group, kind in _GROUPS.items()} - return TargetConfig(**values, **nested) diff --git a/src/sqlbuild/spec/contracts/constants.py b/src/sqlbuild/spec/contracts/constants.py index 917e001590..658c3c4a6a 100644 --- a/src/sqlbuild/spec/contracts/constants.py +++ b/src/sqlbuild/spec/contracts/constants.py @@ -4,8 +4,6 @@ from sqlbuild.spec.contracts.models import SeedCsvSettings -LOADER_SCHEMA_TABLE_PART_COUNT: int = 2 -LOADER_QUALIFIED_TABLE_PART_COUNT: int = 3 DEFAULT_SEED_CSV_SETTINGS: SeedCsvSettings = SeedCsvSettings() CURSOR_POLICY_DISABLED: str = "disabled" ZERO_DAY_CURSOR_DURATION: str = "0d" diff --git a/src/sqlbuild/spec/contracts/main/loader_destination_parts.py b/src/sqlbuild/spec/contracts/main/loader_destination_parts.py deleted file mode 100644 index 7abf45a943..0000000000 --- a/src/sqlbuild/spec/contracts/main/loader_destination_parts.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Loader destination parsing contract.""" - -from __future__ import annotations - -from sqlbuild.spec.contracts.constants import ( - LOADER_QUALIFIED_TABLE_PART_COUNT, - LOADER_SCHEMA_TABLE_PART_COUNT, -) -from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.models import LoaderDestinationParts - - -def loader_destination_parts( - *, destination: str, default_database: str | None, default_schema: str | None -) -> LoaderDestinationParts: - """Parse a one-, two-, or three-part loader destination.""" - - parts: tuple[str, ...] = tuple(destination.split(".")) - if not 1 <= len(parts) <= LOADER_QUALIFIED_TABLE_PART_COUNT or any( - not part.strip() for part in parts - ): - raise SpecConfigError( - f"Invalid loader destination '{destination}': expected 1 to 3 non-empty " - "dot-separated parts (table, schema.table, or database.schema.table)" - ) - if len(parts) == 1: - return LoaderDestinationParts(default_database, default_schema, parts[0]) - if len(parts) == LOADER_SCHEMA_TABLE_PART_COUNT: - return LoaderDestinationParts(default_database, parts[0], parts[1]) - return LoaderDestinationParts(parts[0], parts[1], parts[2]) diff --git a/src/sqlbuild/spec/contracts/main/matching_dynamic_families.py b/src/sqlbuild/spec/contracts/main/matching_dynamic_families.py deleted file mode 100644 index 7f1823b328..0000000000 --- a/src/sqlbuild/spec/contracts/main/matching_dynamic_families.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public compiler entry point for dynamic-family name matching.""" - -from sqlbuild.spec.contracts._helpers.dynamic_columns import matching_dynamic_families as _match -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -def matching_dynamic_families( - *, families: tuple[SchemaDynamicColumnFamily, ...], column_name: str -) -> tuple[SchemaDynamicColumnFamily, ...]: - """Return declared dynamic families accepting a physical column name.""" - - return _match(families=families, column_name=column_name) diff --git a/src/sqlbuild/spec/contracts/main/resolve_effective_adapter_name.py b/src/sqlbuild/spec/contracts/main/resolve_effective_adapter_name.py deleted file mode 100644 index 94e0679659..0000000000 --- a/src/sqlbuild/spec/contracts/main/resolve_effective_adapter_name.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Public effective adapter name resolution operation.""" - -from sqlbuild.spec.contracts._helpers.project_config import ( - resolve_effective_adapter_name as _resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -def resolve_effective_adapter_name( - *, project_config: ProjectConfig, local_config: LocalConfig -) -> str: - """Resolve the effective adapter name, allowing local override.""" - - return _resolve_effective_adapter_name( - project_config=project_config, - local_config=local_config, - ) diff --git a/src/sqlbuild/spec/contracts/main/resolve_effective_collection_rendering.py b/src/sqlbuild/spec/contracts/main/resolve_effective_collection_rendering.py deleted file mode 100644 index 0a978fdcc3..0000000000 --- a/src/sqlbuild/spec/contracts/main/resolve_effective_collection_rendering.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public effective collection rendering resolution operation.""" - -from sqlbuild.spec.contracts._helpers.project_config import ( - resolve_effective_collection_rendering as _resolve_effective_collection_rendering, -) -from sqlbuild.spec.contracts.models import ProjectConfig -from sqlbuild.sql_values.types import CollectionRendering - - -def resolve_effective_collection_rendering( - *, - project_config: ProjectConfig, - declaration_override: CollectionRendering | None, -) -> CollectionRendering: - """Resolve declaration rendering over project configuration.""" - - return _resolve_effective_collection_rendering( - project_config=project_config, - declaration_override=declaration_override, - ) diff --git a/src/sqlbuild/spec/contracts/main/resolve_effective_scenario_config.py b/src/sqlbuild/spec/contracts/main/resolve_effective_scenario_config.py deleted file mode 100644 index 277516fc40..0000000000 --- a/src/sqlbuild/spec/contracts/main/resolve_effective_scenario_config.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Public effective scenario configuration resolution operation.""" - -from sqlbuild.spec.contracts._helpers.project_config import ( - resolve_effective_scenario_config as _resolve_effective_scenario_config, -) -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, ScenarioConfig - - -def resolve_effective_scenario_config( - *, project_config: ProjectConfig, local_config: LocalConfig -) -> ScenarioConfig: - """Resolve scenario config, allowing local overrides to replace project rules.""" - - return _resolve_effective_scenario_config( - project_config=project_config, - local_config=local_config, - ) diff --git a/src/sqlbuild/spec/contracts/main/resolve_target_config.py b/src/sqlbuild/spec/contracts/main/resolve_target_config.py deleted file mode 100644 index 341799dc4f..0000000000 --- a/src/sqlbuild/spec/contracts/main/resolve_target_config.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Public target configuration resolution operation.""" - -from sqlbuild.spec.contracts._helpers.targets import ( - resolve_target_config as _resolve_target_config, -) -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, TargetConfig - - -def resolve_target_config( - *, - project_config: ProjectConfig, - local_config: LocalConfig, - target_name: str, -) -> TargetConfig: - """Merge project target config with local developer overrides.""" - - return _resolve_target_config( - project_config=project_config, - local_config=local_config, - target_name=target_name, - ) diff --git a/src/sqlbuild/spec/contracts/main/resolve_target_name.py b/src/sqlbuild/spec/contracts/main/resolve_target_name.py deleted file mode 100644 index f200723b98..0000000000 --- a/src/sqlbuild/spec/contracts/main/resolve_target_name.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Public target name resolution operation.""" - -from sqlbuild.spec.contracts._helpers.targets import resolve_target_name as _resolve_target_name -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -def resolve_target_name( - *, - project_config: ProjectConfig, - local_config: LocalConfig, - selected_target: str | None, -) -> str | None: - """Resolve the effective target name.""" - - return _resolve_target_name( - project_config=project_config, - local_config=local_config, - selected_target=selected_target, - ) diff --git a/src/sqlbuild/spec/contracts/main/scenario_local_type_overrides_for_dialect.py b/src/sqlbuild/spec/contracts/main/scenario_local_type_overrides_for_dialect.py deleted file mode 100644 index d397a39e27..0000000000 --- a/src/sqlbuild/spec/contracts/main/scenario_local_type_overrides_for_dialect.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Public scenario local type override selection operation.""" - -from sqlbuild.spec.contracts._helpers.project_config import ( - scenario_local_type_overrides_for_dialect as _scenario_local_type_overrides_for_dialect, -) -from sqlbuild.spec.contracts.models import ScenarioConfig - - -def scenario_local_type_overrides_for_dialect( - *, scenario_config: ScenarioConfig, sql_analysis_dialect: str | None -) -> dict[str, str]: - """Return global and dialect-specific scenario local type override rules.""" - - return _scenario_local_type_overrides_for_dialect( - scenario_config=scenario_config, - sql_analysis_dialect=sql_analysis_dialect, - ) diff --git a/src/sqlbuild/sql_values/_helpers/state_literal.py b/src/sqlbuild/sql_values/_helpers/state_literal.py deleted file mode 100644 index 1f78d80dfc..0000000000 --- a/src/sqlbuild/sql_values/_helpers/state_literal.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Implementation helpers for typed warehouse-state SQL literals.""" - -from __future__ import annotations - -import json -from datetime import UTC, date, datetime - -from sqlbuild.sql_values.exceptions import SqlValueRenderingError -from sqlbuild.sql_values.types import StateSqlValueType - - -def render_state_sql_literal(*, value: object | None, declared_type: StateSqlValueType) -> str: - """Render a Python value according to its declared state-column type.""" - - if value is None: - return _typed_null(declared_type=declared_type) - match declared_type: - case StateSqlValueType.STRING: - if not isinstance(value, str): - raise SqlValueRenderingError("state string columns require str values") - return _quoted(value=value) - case StateSqlValueType.INTEGER: - if isinstance(value, bool) or not isinstance(value, int): - raise SqlValueRenderingError("state integer columns require int values") - return f"CAST({value} AS BIGINT)" - case StateSqlValueType.BOOLEAN: - if not isinstance(value, bool): - raise SqlValueRenderingError("state boolean columns require bool values") - return "TRUE" if value else "FALSE" - case StateSqlValueType.TIMESTAMP | StateSqlValueType.TEXT_TIMESTAMP: - if not isinstance(value, datetime): - raise SqlValueRenderingError("state timestamp columns require datetime values") - normalized: datetime = ( - value.replace(tzinfo=UTC) - if value.tzinfo is None or value.utcoffset() is None - else value.astimezone(UTC) - ) - rendered: str = _quoted(value=normalized.isoformat()) - if declared_type == StateSqlValueType.TEXT_TIMESTAMP: - return rendered - return f"CAST({rendered} AS TIMESTAMP)" - case StateSqlValueType.DATE: - if isinstance(value, datetime) or not isinstance(value, date): - raise SqlValueRenderingError("state date columns require date values") - return f"CAST({_quoted(value=value.isoformat())} AS DATE)" - case StateSqlValueType.JSON: - try: - serialized: str = json.dumps( - value, ensure_ascii=False, separators=(",", ":"), sort_keys=True - ) - except (TypeError, ValueError) as exc: - raise SqlValueRenderingError( - "state JSON columns require JSON-serializable values" - ) from exc - return _quoted(value=serialized) - - -def _quoted(*, value: str) -> str: - return "'" + value.replace("'", "''") + "'" - - -def _typed_null(*, declared_type: StateSqlValueType) -> str: - match declared_type: - case StateSqlValueType.INTEGER: - return "CAST(NULL AS BIGINT)" - case StateSqlValueType.TIMESTAMP: - return "CAST(NULL AS TIMESTAMP)" - case StateSqlValueType.DATE: - return "CAST(NULL AS DATE)" - case _: - return "NULL" diff --git a/src/sqlbuild/sql_values/main/normalize.py b/src/sqlbuild/sql_values/main/normalize.py deleted file mode 100644 index bb375ae989..0000000000 --- a/src/sqlbuild/sql_values/main/normalize.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Public entry point for typed SQL value normalization.""" - -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.sql_values.models import SqlValue, SqlValueLimits - - -def normalize_sql_value( - *, - raw_value: object, - context: str, - explicit_type: str | None = None, - limits: SqlValueLimits | None = None, -) -> SqlValue: - """Validate and normalize one authored value with contextual diagnostics.""" - - applied: SqlValueLimits = SqlValueLimits() if limits is None else limits - return cast( - SqlValue, - _native.normalize_typed_sql_value( - raw_value, - (context, explicit_type), - (applied.max_depth, applied.max_elements, applied.max_size), - ), - ) diff --git a/src/sqlbuild/sql_values/main/render_state_literal.py b/src/sqlbuild/sql_values/main/render_state_literal.py deleted file mode 100644 index 222b9177a7..0000000000 --- a/src/sqlbuild/sql_values/main/render_state_literal.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public entry point for portable typed warehouse-state SQL literals.""" - -from sqlbuild.sql_values._helpers.state_literal import ( - render_state_sql_literal as _render_state_sql_literal, -) -from sqlbuild.sql_values.types import StateSqlValueType - - -def render_state_sql_literal(*, value: object | None, declared_type: StateSqlValueType) -> str: - """Render a Python value according to its declared state-column type.""" - - return _render_state_sql_literal(value=value, declared_type=declared_type) diff --git a/src/sqlbuild/sql_values/main/validate_rendered_size.py b/src/sqlbuild/sql_values/main/validate_rendered_size.py deleted file mode 100644 index 8218fcaa3b..0000000000 --- a/src/sqlbuild/sql_values/main/validate_rendered_size.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Rendered typed SQL value size validation.""" - -from sqlbuild.sql_values.constants import DEFAULT_MAX_SQL_VALUE_SIZE -from sqlbuild.sql_values.exceptions import SqlValueValidationError - - -def validate_rendered_sql_value_size( - *, - rendered_sql: str, - context: str, - max_size: int = DEFAULT_MAX_SQL_VALUE_SIZE, -) -> None: - """Reject adapter SQL that exceeds the shared UTF-8 byte limit.""" - - rendered_size: int = len(rendered_sql.encode("utf-8")) - if rendered_size > max_size: - raise SqlValueValidationError( - f"{context} rendered value is {rendered_size} bytes; maximum is {max_size} bytes" - ) diff --git a/src/sqlbuild/compiler/manifest/main/__init__.py b/tests/e2e/scripts/planner_differential/__init__.py similarity index 100% rename from src/sqlbuild/compiler/manifest/main/__init__.py rename to tests/e2e/scripts/planner_differential/__init__.py diff --git a/tests/e2e/scripts/planner_differential/_test_types.py b/tests/e2e/scripts/planner_differential/_test_types.py new file mode 100644 index 0000000000..73026a02fa --- /dev/null +++ b/tests/e2e/scripts/planner_differential/_test_types.py @@ -0,0 +1,20 @@ +"""Test case types for the planner differential end-to-end tests.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class RecordCheckTestCase: + """A record run, a check of the same tree, and a check against a tampered baseline.""" + + description: str + project: str + steps: tuple[str, ...] + tampered_step_index: int + expected_artifacts: tuple[str, ...] + expected_clean_line: str + expected_difference_line: str + expected_diff_line: str + expected_note: str diff --git a/tests/e2e/scripts/planner_differential/helpers.py b/tests/e2e/scripts/planner_differential/helpers.py new file mode 100644 index 0000000000..cfa137836c --- /dev/null +++ b/tests/e2e/scripts/planner_differential/helpers.py @@ -0,0 +1,44 @@ +"""Harness arguments and baseline edits for the planner differential tests.""" + +from __future__ import annotations + +import json +from pathlib import Path + + +def harness_arguments( + *, mode: str, baseline_dir: Path, project: str, steps: tuple[str, ...] +) -> list[str]: + """Return arguments running one corpus project through some steps of the script.""" + + return [ + mode, + "--baseline-dir", + str(baseline_dir), + "--projects", + project, + "--seeds", + "0", + "--jobs", + "1", + "--steps", + *steps, + ] + + +def baseline_steps(path: Path) -> list[dict[str, object]]: + """Return a recorded project's step records.""" + + steps: object = json.loads(path.read_text(encoding="utf-8"))["steps"] + assert isinstance(steps, list) + return steps + + +def tamper_plan_exit_code(*, path: Path, step_index: int) -> None: + """Change one step's recorded `sqb plan --json` exit code, as a planner change would.""" + + document: dict[str, object] = json.loads(path.read_text(encoding="utf-8")) + records: object = document["steps"] + assert isinstance(records, list) + records[step_index]["plan_json"]["exit_code"] = 7 + _ = path.write_text(json.dumps(document), encoding="utf-8") diff --git a/tests/e2e/scripts/planner_differential/test_planner_differential.py b/tests/e2e/scripts/planner_differential/test_planner_differential.py new file mode 100644 index 0000000000..a5772cc81a --- /dev/null +++ b/tests/e2e/scripts/planner_differential/test_planner_differential.py @@ -0,0 +1,64 @@ +"""Record a stateful planner baseline, check the same tree, and catch a changed plan.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.planner_differential.main.differential import run_planner_differential +from tests.e2e.scripts.planner_differential._test_types import RecordCheckTestCase +from tests.e2e.scripts.planner_differential.helpers import ( + baseline_steps, + harness_arguments, + tamper_plan_exit_code, +) + + +@pytest.mark.parametrize( + "test_case", + [ + RecordCheckTestCase( + description="edited_model_on_a_built_project", + project="example/tidy-shop", + steps=("initial", "edit-query"), + tampered_step_index=1, + expected_artifacts=("plan_json", "plan_output", "plan_text", "build", "state"), + expected_clean_line="Planner differential passed: 1 projects matched the baseline", + expected_difference_line=" - step edit-query: plan_json", + expected_diff_line='+ "exit_code": 0,', + expected_note="edited models/daily_revenue.sql", + ), + ], + ids=lambda case: case.description, +) +def test_given_recorded_baseline_when_checking_then_only_a_changed_plan_differs( + test_case: RecordCheckTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + baseline_dir: Path = tmp_path / "baseline" + record_arguments: list[str] = harness_arguments( + mode="record", baseline_dir=baseline_dir, project=test_case.project, steps=test_case.steps + ) + check_arguments: list[str] = harness_arguments( + mode="check", baseline_dir=baseline_dir, project=test_case.project, steps=test_case.steps + ) + + assert run_planner_differential(record_arguments) == 0 + baseline: Path = baseline_dir / f"{test_case.project.replace('/', '__')}.json" + edited: dict[str, object] = baseline_steps(baseline)[-1] + assert edited["mutation"] == {"applied": True, "note": test_case.expected_note} + assert all(edited[artifact] for artifact in test_case.expected_artifacts) + _ = capsys.readouterr() + + assert run_planner_differential(check_arguments) == 0 + assert test_case.expected_clean_line in capsys.readouterr().out + + tamper_plan_exit_code(path=baseline, step_index=test_case.tampered_step_index) + assert run_planner_differential(check_arguments) == 1 + output: str = capsys.readouterr().out + assert test_case.expected_difference_line in output + assert test_case.expected_diff_line in output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 807be35a12..70f7e9f404 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -33,10 +33,7 @@ import sqlbuild.adapter.type_system._helpers.type_normalization as type_normalization import sqlbuild.cli.compile_reuse._helpers.native_reuse as native_reuse import sqlbuild.cli.compile_reuse._helpers.provider_settings as provider_settings -import sqlbuild.compiler.compile._helpers.diagnostics.recovery as diagnostic_recovery import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module -import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stages -import sqlbuild.compiler.compile._helpers.native_stages.sql_tests as native_sql_test_stage import sqlbuild.compiler.frontier.main.compiled_code_identity as compiled_code_identity_module import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class from scripts.cold_compile_performance.main.read_compile_measurement import read_compile_measurement @@ -4811,11 +4808,11 @@ def full_macro_call_runs( _ANALYSIS_SEAMS: tuple[tuple[ModuleType, str], ...] = ( - (native_stages, "assemble_native_project_resources"), - (native_stages, "infer_native_expression_source_shapes"), - (native_stages, "analyze_native_model_sql"), - (native_sql_test_stage, "assemble_native_sql_tests"), - (diagnostic_recovery, "complete_semantic_project"), + (native_module, "assemble_native_project_resources"), + (native_module, "infer_native_expression_source_shapes"), + (native_module, "analyze_native_model_sql"), + (native_module, "assemble_native_sql_tests"), + (native_module, "complete_semantic_project"), (native_module, "evaluate_model_contracts"), (native_module, "promotion_conflict_diagnostics"), (native_module, "build_project_column_lineage"), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index 06c9e50f03..ab8d8c437d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -30,7 +30,7 @@ description="models_contracts_lineage_and_sql_tests", expected_native_returns={ "assemble_native_project_resources": [ANY], - "infer_native_expression_source_shapes": [ANY], + "infer_native_expression_source_shapes": [ANY, ANY, ANY, ANY], "analyze_native_model_sql": [ANY], "assemble_native_sql_tests": [ANY], "complete_semantic_project": [ANY], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_config_errors.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_config_errors.py index 90ddd1f640..c893b37765 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_config_errors.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_config_errors.py @@ -7,7 +7,7 @@ import pytest -import sqlbuild.compiler.compile._helpers.render.context_templates as context_templates +import sqlbuild._native as _native from scripts.compiler_differential.constants import FAILURE_BASE_MART from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( MacroExpandedValidatorErrorTestCase, @@ -43,7 +43,7 @@ " materialized incremental,\n incremental_strategy append,\n cursor order_id,\n" " cursor_type integer,\n" ) -_FALLBACKS: tuple[tuple[ModuleType, str], ...] = ((context_templates, "expand_template_data"),) +_FALLBACKS: tuple[tuple[ModuleType, str], ...] = ((_native, "expand_template_data"),) _PROJECT_FILES: dict[str, str] = { "macros/counted.py": _COUNTED_MACRO, "models/marts/customer_totals.sql": _COUNTED_MART, diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/test_typed_sql_rendering.py b/tests/integration/src/sqlbuild/adapters/duckdb/test_typed_sql_rendering.py index 2804239138..13886078d3 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/test_typed_sql_rendering.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/test_typed_sql_rendering.py @@ -3,8 +3,8 @@ import duckdb import pytest +from sqlbuild._native import normalize_sql_value from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.sql_values.main.normalize import normalize_sql_value from tests.integration.src.sqlbuild.adapters.duckdb._test_types import ( TypedSqlRenderingExecutionTestCase, ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py index e5df8a4dc9..9cba7be2a3 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py @@ -41,7 +41,6 @@ def test_given_plain_compile_when_running_fresh_cli_then_optional_artifact_impor ]) Path(sys.argv[2]).write_text(json.dumps({ "aggregate_models": "sqlbuild.cli.commands.models.runtime" in sys.modules, - "manifest": "sqlbuild.compiler.manifest.main.build" in sys.modules, }), encoding="utf-8") raise SystemExit(exit_code) """ @@ -58,7 +57,6 @@ def test_given_plain_compile_when_running_fresh_cli_then_optional_artifact_impor assert summary["models"] == test_case.expected_count assert json.loads(import_status_path.read_text(encoding="utf-8")) == { "aggregate_models": False, - "manifest": False, } diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 87b1d933fb..08b31e3398 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -13,7 +13,6 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.render.user_macros as macros_module -import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module import sqlbuild.rule_engine._helpers.host.custom_host_pool as custom_host_pool_module import sqlbuild.rule_engine._helpers.run.rule_threads as rule_threads_module import sqlbuild.rule_engine._helpers.run.rules as rules_module @@ -834,7 +833,6 @@ def counting_build( ) monkeypatch.setattr(native_module, "build_declaration_scope", counting_build) - monkeypatch.setattr(expansion_context_module, "build_declaration_scope", counting_build) warm: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) @@ -874,9 +872,7 @@ def test_given_cold_scoped_macro_sql_test_when_compiling_then_early_lint_uses_pr scope_builds: list[Path] = [] macro_loads: list[dict[str, LoadedMacro]] = [] original_build: Callable[..., DeclarationScopeBuild] = native_module.build_declaration_scope - original_load: Callable[..., dict[str, LoadedMacro]] = ( - expansion_context_module.load_project_macros - ) + original_load: Callable[..., dict[str, LoadedMacro]] = macros_module.load_project_macros def counting_build( *, @@ -899,9 +895,7 @@ def recording_load(macro_files: tuple[DiscoveredMacroFile, ...]) -> dict[str, Lo return loaded monkeypatch.setattr(native_module, "build_declaration_scope", counting_build) - monkeypatch.setattr(expansion_context_module, "build_declaration_scope", counting_build) monkeypatch.setattr(macros_module, "load_project_macros", recording_load) - monkeypatch.setattr(expansion_context_module, "load_project_macros", recording_load) cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 3594df7e8a..cc5bc10b8b 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -14,11 +14,10 @@ import pytest import sqlbuild._native as native_module -import sqlbuild.compiler.compile._helpers.diagnostics.recovery as semantic_recovery -import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly from sqlbuild._native import ( analyze_native_model_sql, build_compile_inputs, + cursor_intrinsics_analysis_sql, discover_project_inputs, infer_native_expression_source_shapes, model_placeholders, @@ -38,9 +37,6 @@ ) from sqlbuild.compiler.analysis_session.types import ProofRow from sqlbuild.compiler.compile._helpers.analysis.cache import build_analysis_cache_context -from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( - cursor_intrinsics_analysis_sql, -) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( AnalysisCacheContext, @@ -420,7 +416,7 @@ def native_pivot_proofs(*, monkeypatch: pytest.MonkeyPatch) -> NativePivotProofs recorded_proofs: NativePivotProofs = NativePivotProofs() native_proofs: Callable[..., tuple[ProofRow | None, ...]] = ( - native_stage_assembly.prove_native_dynamic_contracts + native_module.prove_native_dynamic_contracts ) def recorded(**keywords: Any) -> tuple[ProofRow | None, ...]: @@ -430,7 +426,7 @@ def recorded(**keywords: Any) -> tuple[ProofRow | None, ...]: recorded_proofs.session_proofs += proven * (keywords["session"] is not None) return proofs - monkeypatch.setattr(native_stage_assembly, "prove_native_dynamic_contracts", recorded) + monkeypatch.setattr(native_module, "prove_native_dynamic_contracts", recorded) return recorded_proofs @@ -589,7 +585,7 @@ def analyse_natively( `runs.golden` gathers the entries the Python oracle recorded for the same seam calls. """ - original_analysis: Callable[..., NativeModelAnalyses] = native_stage_assembly.analyze_model_sql + original_analysis: Callable[..., NativeModelAnalyses] = analyze_native_model_sql def analysed_three_ways(request: NativeModelAnalysisRequest) -> NativeModelAnalyses: profile: ExpressionInferenceProfile = request.inference_profile @@ -609,7 +605,9 @@ def analysed_three_ways(request: NativeModelAnalysisRequest) -> NativeModelAnaly cached_views.append( (_analysis_views(_projected(cached, request, runs)), _catalog_view(catalog)) ) - native: NativeModelAnalyses = original_analysis(replace(request, analysis_cache=None)) + native: NativeModelAnalyses = original_analysis( + request=replace(request, analysis_cache=None) + ) session: Any = native.session analyses: dict[str, ModelSqlAnalysis] = _projected(native, request, runs) uncached_view: object = (_analysis_views(analyses), _catalog_view(profile.binding_catalog)) @@ -636,18 +634,24 @@ def recorded_shapes( runs.golden.append(golden_entry("shapes", (native, catalog.expression_shapes))) return native - completed: Callable[..., CompiledProjectView] = semantic_recovery.complete_semantic_diagnostics + completed: Callable[..., CompiledProjectView] = native_module.complete_semantic_project def recorded_proofs(*, project: CompiledProjectView, **keywords: Any) -> CompiledProjectView: runs.golden.extend( golden_entry("proof", model.dynamic_column_contract) for model in project.models ) - return completed(project=project, **keywords) + with monkeypatch.context() as completing: + completing.setattr( + native_module, + "infer_native_expression_source_shapes", + infer_native_expression_source_shapes, + ) + return completed(project=project, **keywords) with monkeypatch.context() as patch: - patch.setattr(semantic_recovery, "complete_semantic_diagnostics", recorded_proofs) - patch.setattr(native_stage_assembly, "analyze_model_sql", analysed_three_ways) - patch.setattr(native_stage_assembly, "expression_source_shapes", recorded_shapes) + patch.setattr(native_module, "complete_semantic_project", recorded_proofs) + patch.setattr(native_module, "analyze_native_model_sql", analysed_three_ways) + patch.setattr(native_module, "infer_native_expression_source_shapes", recorded_shapes) with suppress(CompileInputError): _ = native_module.assemble_project( inputs=inputs, @@ -693,7 +697,7 @@ def captured(request: NativeModelAnalysisRequest) -> NativeModelAnalyses: monkeypatch.context() as patch, suppress(CompileInputError, native_module.NativeCompilerError), ): - patch.setattr(native_stage_assembly, "analyze_model_sql", captured) + patch.setattr(native_module, "analyze_native_model_sql", captured) _ = native_module.assemble_project( inputs=inputs, inference_profile=inference_profile, diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py index 82852ea94d..2bf800b5c8 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py @@ -12,7 +12,6 @@ import pytest import sqlbuild._native as native_module -import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly from sqlbuild._native import infer_native_query_columns from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeModelAnalyses @@ -130,9 +129,9 @@ def test_given_session_without_model_analysis_when_assembling_then_raises_native test_case: MissingSessionAnalysisTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr( - native_stage_assembly, - "analyze_model_sql", - lambda _request: NativeModelAnalyses(positions={}, succeeded=frozenset(), session=None), + native_module, + "analyze_native_model_sql", + lambda request: NativeModelAnalyses(positions={}, succeeded=frozenset(), session=None), ) with pytest.raises(native_module.NativeCompilerError) as raised: diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py index 9607dae950..e28c5e327a 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -21,6 +21,7 @@ expand_sql_macros, mask_capture_noise, render_stage_capture, + resolve_effective_collection_rendering, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -37,9 +38,6 @@ from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE from sqlbuild.compiler.scopes.models import ResourceIdentity from sqlbuild.compiler.scopes.types import ResourceKind -from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( - resolve_effective_collection_rendering, -) MACRO_CALL_LOG_ENV_VAR: str = "SQLBUILD_TEST_MACRO_CALL_LOG" _RUN_ID_PATTERN: re.Pattern[str] = re.compile(r'"run_id": "[^"]*"') diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index aafc9018a9..11c98422f9 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -12,15 +12,17 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import build_compile_inputs, discover_project_inputs, parse_audit_instances +from sqlbuild._native import ( + build_compile_inputs, + discover_project_inputs, + parse_audit_instances, + resolve_effective_collection_rendering, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileAdapterContext from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile -from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( - resolve_effective_collection_rendering, -) from sqlbuild.spec.contracts.models import SchemaModelEntry, SourceLocation _TEXTS: tuple[object, ...] = ( @@ -107,7 +109,7 @@ ) -NATIVE_MODEL_CONFIG_ENTRIES: tuple[str, ...] = ("expand_config_templates",) +NATIVE_MODEL_CONFIG_ENTRIES: tuple[str, ...] = ("expand_template_data",) _NATIVE_ENTRIES: dict[str, Callable[..., object]] = { name: getattr(_native, name) for name in NATIVE_MODEL_CONFIG_ENTRIES } diff --git a/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py index 70bd076c17..d7750bb698 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py @@ -97,7 +97,7 @@ def test_given_template_or_macro_column_metadata_when_attaching_then_loop_expand ModelConfigTierTestCase( description="native model config", expected_native_calls={ - "expand_config_templates": 0, + "expand_template_data": 0, }, ), ], diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index f3fd696ec9..f713823e15 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -9,10 +9,10 @@ import pytest import sqlbuild._native as native_module -import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly from sqlbuild._native import ( build_compile_inputs, build_compiled_project, + build_manifest, discover_project_inputs, run_compile_pipeline, semantic_shapes, @@ -27,7 +27,6 @@ ) from sqlbuild.compiler.compile.types import AuditInput, CompiledModelView, CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.sql_values.types import CollectionRendering @@ -218,13 +217,12 @@ def traced_shape_batches(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, ... """Record each shape request's expressions the binding catalog's shape cache misses.""" batches: list[tuple[str, ...]] = [] - for module in (native_stage_assembly, native_module): - original: Callable[..., Any] = module.infer_native_expression_source_shapes - monkeypatch.setattr( - module, - "infer_native_expression_source_shapes", - functools.partial(_traced_shapes, original=original, batches=batches), - ) + original: Callable[..., Any] = native_module.infer_native_expression_source_shapes + monkeypatch.setattr( + native_module, + "infer_native_expression_source_shapes", + functools.partial(_traced_shapes, original=original, batches=batches), + ) return batches diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py index 9d44085150..8e58afe6ad 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py @@ -6,12 +6,15 @@ import pytest -from sqlbuild._native import build_compiled_project, discover_project_inputs +import sqlbuild.compiler.sql_analysis.models as sql_analysis_models +from sqlbuild._native import ( + build_compiled_project, + discover_project_inputs, + source_binding_diagnostics, +) from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile._helpers.analysis import binding_requests -from sqlbuild.compiler.compile.main._source_bindings import get_source_binding_diagnostics from sqlbuild.compiler.compile.models import CompiledObjectKey, CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( @@ -123,19 +126,19 @@ def test_given_catalog_when_rebinding_physical_shapes_then_does_not_reuse_stale_ profile: ExpressionInferenceProfile = ExpressionInferenceProfile( sql_analysis_dialect="snowflake" ) - missing: tuple[CompilerDiagnostic, ...] = get_source_binding_diagnostics( + missing: tuple[CompilerDiagnostic, ...] = source_binding_diagnostics( project=project, columns={"orders": (ColumnInfo("customer", "VARCHAR"),)}, profile=profile, selected_keys=keys, ) - present: tuple[CompilerDiagnostic, ...] = get_source_binding_diagnostics( + present: tuple[CompilerDiagnostic, ...] = source_binding_diagnostics( project=project, columns={"orders": (ColumnInfo("OrderId", "INTEGER"),)}, profile=profile, selected_keys=keys, ) - repeated: tuple[CompilerDiagnostic, ...] = get_source_binding_diagnostics( + repeated: tuple[CompilerDiagnostic, ...] = source_binding_diagnostics( project=project, columns={"orders": (ColumnInfo("customer", "VARCHAR"),)}, profile=profile, @@ -177,7 +180,7 @@ def test_given_sql_when_case_policy_enabled_then_fused_validation_is_reused( def repeated_binding(**kwargs: Any) -> None: raise AssertionError("already validated SQL was normalized and bound again") - monkeypatch.setattr(binding_requests, "get_complete_schema_binding_request", repeated_binding) + monkeypatch.setattr(sql_analysis_models, "SqlSchemaValidationRequest", repeated_binding) assert ( main( [ diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py index 56176ec05b..f6d98e54f3 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py @@ -8,11 +8,14 @@ import pytest import sqlbuild._native as native -from sqlbuild._native import build_compiled_project, discover_project_inputs +from sqlbuild._native import ( + build_compiled_project, + discover_project_inputs, + source_binding_diagnostics, +) from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile.main._source_bindings import get_source_binding_diagnostics from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( @@ -99,7 +102,7 @@ def test_given_effective_target_when_binding_quotes_then_honours_session_setting adapter=SnowflakeAdapter(), selected_target="selected", ) - physical: tuple[CompilerDiagnostic, ...] = get_source_binding_diagnostics( + physical: tuple[CompilerDiagnostic, ...] = source_binding_diagnostics( project=project, columns={"orders": (ColumnInfo(name="customer", type="VARCHAR"),)}, profile=ExpressionInferenceProfile(sql_analysis_dialect="snowflake"), diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index 295ba0a2c1..79c0b1d7c9 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -60,7 +60,6 @@ class PlannerChangeDetectionWorkTestCase: expected_model_calls: dict[str, int] expected_warning_count: int query_change_tracking: bool = True - expected_query_tracking_calls: dict[tuple[str, bool], int] = field(default_factory=dict) expected_identity_builds: int = 1 diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py index 323774a72f..fe34b95e93 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py @@ -15,7 +15,7 @@ from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.changes import detect as change_detection +from sqlbuild.compiler.planner._helpers.identity import model_metadata from sqlbuild.compiler.planner._helpers.planning import identities as identity_planning from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( @@ -66,17 +66,12 @@ selection_diagnostics=False, expected_model_calls={"orders": 1}, expected_warning_count=0, - expected_query_tracking_calls={("orders", True): 1}, ), PlannerChangeDetectionWorkTestCase( description="selection diagnostics enabled", selection_diagnostics=True, expected_model_calls={"stg_orders": 1, "orders": 1}, expected_warning_count=0, - expected_query_tracking_calls={ - ("stg_orders", True): 1, - ("orders", True): 1, - }, expected_identity_builds=2, ), PlannerChangeDetectionWorkTestCase( @@ -85,11 +80,6 @@ expected_model_calls={"stg_orders": 1, "orders": 2}, expected_warning_count=0, query_change_tracking=False, - expected_query_tracking_calls={ - ("orders", False): 1, - ("stg_orders", True): 1, - ("orders", True): 1, - }, expected_identity_builds=2, ), ], @@ -129,8 +119,8 @@ def test_given_selected_model_when_planning_then_each_required_model_is_change_d query_change_tracking=test_case.query_change_tracking, ), ) - model_detector: Mock = Mock(wraps=change_detection.detect_model_changes) - monkeypatch.setattr(change_detection, "detect_model_changes", model_detector) + model_detector: Mock = Mock(wraps=model_metadata.declared_columns_hash) + monkeypatch.setattr(model_metadata, "declared_columns_hash", model_detector) identity_builder: Mock = Mock(wraps=identity_planning.build_direct_model_version_identities) monkeypatch.setattr( identity_planning, @@ -150,11 +140,6 @@ def test_given_selected_model_when_planning_then_each_required_model_is_change_d call.kwargs["model"].name for call in model_detector.call_args_list ) assert dict(model_calls) == test_case.expected_model_calls - query_tracking_calls: Counter[tuple[str, bool]] = Counter( - (call.kwargs["model"].name, call.kwargs["query_change_tracking"]) - for call in model_detector.call_args_list - ) - assert dict(query_tracking_calls) == test_case.expected_query_tracking_calls assert identity_builder.call_count == test_case.expected_identity_builds assert len(plan.warnings) == test_case.expected_warning_count diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py index cd500ba70c..17fd7216d5 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py @@ -52,17 +52,6 @@ class OptOutTestCase: expected_rejected: dict[str, bool] -@dataclass(frozen=True) -class GraphFactsTestCase: - """A project whose assembly request carries each resource's project-graph facts.""" - - description: str - files: dict[str, str] - expected_model_tags: tuple[list[str], ...] - expected_seeds: list[tuple[str, list[str]]] - expected_function_kinds: tuple[str, ...] - - @dataclass(frozen=True) class NativePanicTestCase: """A project whose native assembly panics in the parser.""" diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py index 2b3f192183..4c3a914582 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -25,7 +25,6 @@ CompileAdapterContext, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.project_assembly.types import ProjectRequestRow from sqlbuild.sql_values.types import CollectionRendering _PYTHON_RESOURCE_FUNCTIONS: tuple[str, ...] = ("validate_sql_syntax", "validate_hook_sql_syntax") @@ -244,20 +243,6 @@ def python_resource_calls(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: return calls -def record_assembly_requests(*, monkeypatch: pytest.MonkeyPatch) -> list[ProjectRequestRow]: - """Record every request the native project assembly receives.""" - - requests: list[ProjectRequestRow] = [] - original: Callable[..., Any] = _native.assemble_project_resource_facts - - def recording(request: ProjectRequestRow) -> Any: - requests.append(request) - return original(request) - - monkeypatch.setattr(_native, "assemble_project_resource_facts", recording) - return requests - - def _write(*, path: Path, contents: str) -> int: path.parent.mkdir(parents=True, exist_ok=True) return path.write_text(contents, encoding="utf-8") diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py index 795b552976..652a09570b 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py @@ -14,7 +14,6 @@ from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.project_assembly.types import ProjectRequestRow from tests.integration.src.sqlbuild.compiler.helpers import ( GoldenEntry, golden_differences, @@ -26,7 +25,6 @@ AssemblyDeferralTestCase, EdgeAssemblyTestCase, GeneratedAssemblyParityTestCase, - GraphFactsTestCase, NativePanicTestCase, OptOutTestCase, WindowsEnvironmentTestCase, @@ -38,7 +36,6 @@ generated_assembly_files, project_inputs, python_resource_calls, - record_assembly_requests, recorded_assembly, rejected_opt_outs, seed_yml, @@ -251,68 +248,6 @@ def test_given_sql_analysis_opt_outs_when_attaching_natively_then_python_rejecti assert native == test_case.expected_rejected -@pytest.mark.parametrize( - "test_case", - [ - GraphFactsTestCase( - description="tagged models in folders, tagged seeds and both function kinds", - files={ - "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', - "models/staging/orders.sql": ( - "MODEL (description 'Orders.', tags [daily, finance]);\n\nSELECT 1 AS id\n" - ), - "models/customers.sql": ( - "MODEL (description 'Customers.');\n\nSELECT id FROM __ref(\"orders\")\n" - ), - "seeds/regions.yml": ( - "seeds:\n - name: regions\n description: Regions.\n" - " tags: [lookup]\n columns:\n - name: n\n type: INTEGER\n" - ), - "seeds/regions.csv": "n\n1\n", - "functions/sql/is_big.sql": ( - "FUNCTION (description 'Big.', arguments (x INTEGER), returns BOOLEAN," - " tags [math]);\n\nx > 100\n" - ), - "functions/sql/order_ids.sql": ( - "FUNCTION (description 'Ids.', arguments (x INTEGER)," - ' returns table (id INTEGER));\n\nSELECT id FROM __ref("orders")\n' - ), - }, - expected_model_tags=([], ["daily", "finance"]), - expected_seeds=[("regions", ["lookup"])], - expected_function_kinds=("udf", "table_fn"), - ) - ], - ids=lambda case: case.description, -) -def test_given_project_when_assembling_natively_then_request_carries_graph_facts( - test_case: GraphFactsTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - requests: list[ProjectRequestRow] = record_assembly_requests(monkeypatch=monkeypatch) - project: CompiledProjectView = assemble_with( - inputs=project_inputs(project_dir=tmp_path / "project", files=test_case.files), - monkeypatch=monkeypatch, - ) - request: ProjectRequestRow = requests[0] - - assert ( - [(name, directory, tags) for name, directory, tags, _, _ in request[5]], - [(name, tags) for name, tags, _, _ in request[7]], - [(kind, name, tags) for kind, name, tags, _ in request[8]], - ) == ( - [ - (model.key.name, str(model.relative_path.parent)) + (tags,) - for model, tags in zip(project.models, test_case.expected_model_tags, strict=True) - ], - test_case.expected_seeds, - [ - (str(function.key.resource_type), function.key.name, list(function.tags)) - for function in project.functions - ], - ) - assert [kind for kind, _, _, _ in request[8]] == list(test_case.expected_function_kinds) - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py b/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py index 5cfa7d863b..ca7ba5884e 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py @@ -34,14 +34,6 @@ class ScopeCommandParityTestCase: expected_minimum_grants: int -@dataclass(frozen=True) -class ScopeEngineTierTestCase: - """How often building the declaration scope opens the native index.""" - - description: str - expected_native_attempts: int - - @dataclass(frozen=True) class ExpectedNameScanTestCase: """Seeded SQL test bodies whose relationship scans match Python's recorded golden.""" diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index fd191cfe21..0a0558e330 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -16,9 +16,11 @@ from sqlbuild._native import ( build_declaration_scope, discover_project_inputs, + resolve_declaration_context, + resolve_scope_declaration_visibility, + resolve_scope_path_visibility, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_context from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( @@ -33,15 +35,7 @@ DiscoveredProjectInputs, EnumDeclaration, ) -from sqlbuild.compiler.scopes.main import _open_native_scope_index as open_index_module -from sqlbuild.compiler.scopes.main._declaration_lexical_path import declaration_lexical_path -from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index -from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( - resolve_scope_declaration_visibility, -) -from sqlbuild.compiler.scopes.main._resolve_scope_path_visibility import ( - resolve_scope_path_visibility, -) +from sqlbuild.compiler.scopes.constants import CURRENT_PATH_COMPONENT, DECLARATION_LEXICAL_STEM from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, @@ -312,32 +306,6 @@ def scope_command_index(*, project_dir: Path) -> ScopeIndex: return _native.load_or_build_scope_index(project_dir=project_dir, no_cache=True) -def native_scope_attempts( - *, project_dir: Path, monkeypatch: pytest.MonkeyPatch -) -> tuple[int, ScopeOutcome]: - """Build the declaration scope; count how often the native index was opened.""" - - discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) - macros: dict[str, LoadedMacro] = load_project_macros(discovered.macro_files) - attempts: list[None] = [] - - def counted( - *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: Mapping[str, LoadedMacro] - ) -> _native.NativeScopeIndex: - attempts.append(None) - return open_native_scope_index( - discovered_inputs=discovered_inputs, loaded_macros=loaded_macros - ) - - monkeypatch.setattr(open_index_module, "open_native_scope_index", counted) - outcome: ScopeOutcome = _scope_outcome( - discovered=discovered, - macros=macros, - syntax=DuckDbAdapter().sql_lexical_syntax, - ) - return len(attempts), outcome - - def _scope_outcome( *, discovered: DiscoveredProjectInputs, @@ -769,7 +737,9 @@ def _path_visibility( _ = resolution lexical_path: Path = next( map( - lambda record: Path(declaration_lexical_path(record=record)), + lambda record: ( + Path(record.owning_path or CURRENT_PATH_COMPONENT) / DECLARATION_LEXICAL_STEM + ), filter( lambda record: ( record.path == target_path.as_posix() and record.scope is not ScopeKind.PRIVATE diff --git a/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py b/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py index c427e57c6c..c0bdd44c72 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py @@ -12,13 +12,11 @@ GeneratedScopeParityTestCase, ScopeCommandParityTestCase, ScopeEngineParityTestCase, - ScopeEngineTierTestCase, ) from tests.integration.src.sqlbuild.compiler.scopes.helpers import ( SCOPED_PROJECT, ScopeOutcome, generated_scope_files, - native_scope_attempts, scope_command_index, scope_outcome, write_project, @@ -198,24 +196,5 @@ def test_given_scoped_project_when_scope_command_builds_index_then_grants_are_ke assert len(index.grants) >= test_case.expected_minimum_grants, test_case.description -@pytest.mark.parametrize( - "test_case", - [ - ScopeEngineTierTestCase( - description="native declaration scopes", expected_native_attempts=1 - ), - ], - ids=lambda case: case.description, -) -def test_given_scoped_project_when_building_scope_then_the_native_index_is_built_once( - test_case: ScopeEngineTierTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - write_project(project_dir=tmp_path, files=SCOPED_PROJECT) - - attempts, outcome = native_scope_attempts(project_dir=tmp_path, monkeypatch=monkeypatch) - - assert (attempts, outcome.kind) == (test_case.expected_native_attempts, "ok") - - if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index c17d00de94..d81bc4f55a 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -13,14 +13,10 @@ import pytest import sqlbuild._native as native_module -import sqlbuild.compiler.compile._helpers.diagnostics.recovery as semantic_recovery from sqlbuild._native import build_project_graph, discover_project_inputs, project_with from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.analysis_session.types import DiagnosticRow -from sqlbuild.compiler.compile._helpers.diagnostics.recovery import ( - complete_semantic_diagnostics, -) from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.integration.src.sqlbuild.compiler.analysis_session.helpers import ( @@ -84,13 +80,13 @@ @dataclass(frozen=True) class SemanticInputs: - """The arguments the Python compile passed to `complete_semantic_diagnostics`.""" + """The arguments the compile passed to `complete_semantic_project`.""" project: CompiledProjectView profile: ExpressionInferenceProfile binding_results: dict[str, list[DiagnosticRow]] resource_sql_analysis: bool - native_session: Any | None = None + session: Any | None = None def _typo(rng: random.Random, column: str) -> str: @@ -196,12 +192,14 @@ def captured_semantic_inputs( _ = path.write_text(contents, encoding="utf-8") captured: list[SemanticInputs] = [] + complete: Callable[..., CompiledProjectView] = native_module.complete_semantic_project + def capture(**arguments: Any) -> CompiledProjectView: captured.append(SemanticInputs(**arguments)) - return complete_semantic_diagnostics(**arguments) + return complete(**arguments) with monkeypatch.context() as patch: - patch.setattr(semantic_recovery, "complete_semantic_diagnostics", capture) + patch.setattr(native_module, "complete_semantic_project", capture) _ = build_project_graph( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), @@ -243,7 +241,7 @@ def native_completion(inputs: SemanticInputs) -> CompiledProjectView: profile=inputs.profile, binding_results=inputs.binding_results, resource_sql_analysis=inputs.resource_sql_analysis, - session=inputs.native_session, + session=inputs.session, ) @@ -271,13 +269,13 @@ def proven_output_count(project: CompiledProjectView) -> int: def without_session(inputs: SemanticInputs) -> SemanticInputs: """The same stage inputs with every model's output names and lineage in the payload.""" - return replace(inputs, native_session=None) + return replace(inputs, session=None) def session_fact_models(inputs: SemanticInputs) -> frozenset[str]: """Models whose output names and lineage the finished session supplies to the stage.""" - kept: frozenset[str] = frozenset(getattr(inputs.native_session, "fact_models", ())) + kept: frozenset[str] = frozenset(getattr(inputs.session, "fact_models", ())) proven: frozenset[str] = frozenset( model.name for model in filter(_output_proven, inputs.project.models) ) diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py index 1ec117546b..37a7d81982 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py @@ -285,7 +285,7 @@ def test_given_compiles_when_completing_from_the_session_then_matches_payload_an session_models: int = sum(len(selected) for selected in selections) model_count: int = sum(len(inputs.project.models) for inputs in captured) - assert all(inputs.native_session is not None for inputs in captured) + assert all(inputs.session is not None for inputs in captured) assert ( golden_differences( read_golden(golden_name("semantic_session", test_case.description)), golden diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py index 94547ebe5c..0332ad8888 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py @@ -12,9 +12,9 @@ import pytest +import sqlbuild._native as native_module from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile._helpers.native_stages import sql_tests as sql_test_stage from sqlbuild.compiler.compile.models import ( CompiledSqlTest, ) @@ -463,9 +463,7 @@ def record_native_assemblies(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str] """ answers: Counter[str] = Counter() - seam: Callable[..., tuple[NativeSqlTestAssembly, ...]] = ( - sql_test_stage.assemble_native_sql_tests - ) + seam: Callable[..., tuple[NativeSqlTestAssembly, ...]] = native_module.assemble_native_sql_tests def counted_assemblies(*, inputs: CompileInputsView) -> tuple[NativeSqlTestAssembly, ...]: assemblies: tuple[NativeSqlTestAssembly, ...] = seam(inputs=inputs) @@ -480,7 +478,7 @@ def counted_assemblies(*, inputs: CompileInputsView) -> tuple[NativeSqlTestAssem ) return assemblies - monkeypatch.setattr(sql_test_stage, "assemble_native_sql_tests", counted_assemblies) + monkeypatch.setattr(native_module, "assemble_native_sql_tests", counted_assemblies) return answers diff --git a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py index 1cce92fb83..b4605751e6 100644 --- a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py +++ b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py @@ -291,7 +291,7 @@ def test_given_freshness_code_when_checking_sql_ownership_then_matches_contract( ), CustomRuleTestCase( description="compile input macro load passes", - path="src/sqlbuild/compiler/compile/main/sql_expansion_context.py", + path="src/sqlbuild/compiler/compile/_helpers/render/user_macros.py", source="loaded = load_project_macros(())\n", expected_fault_count=0, ), diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py index 36668c4da2..36e7280746 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -113,8 +113,11 @@ def test_given_native_source_when_checking_module_paths_then_reports_errors( "sqlbuild.cli.commands._helpers.cost.collection", "sqlbuild.cli.commands.classes.prepared_compile_artifacts", "sqlbuild.compile", + "sqlbuild.executor.runtime", "sqlbuild.fingerprints", "sqlbuild.lineage", + "sqlbuild.planner", + "sqlbuild.planner.changes", ), ), ], diff --git a/src/sqlbuild/spec/contracts/_helpers/__init__.py b/tests/unit/scripts/planner_differential/__init__.py similarity index 100% rename from src/sqlbuild/spec/contracts/_helpers/__init__.py rename to tests/unit/scripts/planner_differential/__init__.py diff --git a/src/sqlbuild/spec/contracts/main/__init__.py b/tests/unit/scripts/planner_differential/_helpers/__init__.py similarity index 100% rename from src/sqlbuild/spec/contracts/main/__init__.py rename to tests/unit/scripts/planner_differential/_helpers/__init__.py diff --git a/src/sqlbuild/sql_values/_helpers/__init__.py b/tests/unit/scripts/planner_differential/_helpers/capturing/__init__.py similarity index 100% rename from src/sqlbuild/sql_values/_helpers/__init__.py rename to tests/unit/scripts/planner_differential/_helpers/capturing/__init__.py diff --git a/tests/unit/scripts/planner_differential/_helpers/capturing/_test_types.py b/tests/unit/scripts/planner_differential/_helpers/capturing/_test_types.py new file mode 100644 index 0000000000..0569f7c71f --- /dev/null +++ b/tests/unit/scripts/planner_differential/_helpers/capturing/_test_types.py @@ -0,0 +1,42 @@ +"""Test case types for the planner differential canonicalisation helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class MaskTextTestCase: + """Raw output text and the text after masking run-specific values.""" + + description: str + raw: str + expected_text: str + + +@dataclass(frozen=True) +class CanonicalValueTestCase: + """A decoded JSON value and its canonical form.""" + + description: str + value: object + expected_value: object + + +@dataclass(frozen=True) +class StateDeltaTestCase: + """State rows before and after a build and the rows the step gained and lost.""" + + description: str + before: dict[str, list[str]] + after: dict[str, list[str]] + expected_delta: dict[str, dict[str, list[object]]] + + +@dataclass(frozen=True) +class PlanCaptureTestCase: + """A value written as a plan capture document and its canonical expanded form.""" + + description: str + value: object + expected_value: object diff --git a/tests/unit/scripts/planner_differential/_helpers/capturing/helpers.py b/tests/unit/scripts/planner_differential/_helpers/capturing/helpers.py new file mode 100644 index 0000000000..d96fa13bad --- /dev/null +++ b/tests/unit/scripts/planner_differential/_helpers/capturing/helpers.py @@ -0,0 +1,59 @@ +"""Masks and plan-like values for the canonicalisation tests.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, date, datetime + +from scripts.planner_differential.models import TextMasks + +RUN_DATE: date = date(2026, 10, 14) +TEST_MASKS: TextMasks = TextMasks( + replacements=( + ("/runs/work/orders/project", ""), + ("/runs/work", ""), + ), + today=RUN_DATE, +) +LONG_QUERY: str = "SELECT order_id FROM orders WHERE status = 'shipped'\n" * 30 + + +@dataclass(frozen=True) +class OrderPlanEntry: + """A plan entry shaped like the planner's: a name, a query and an invocation time.""" + + name: str + query: str + invocation_time: datetime + data_start: datetime + + +@dataclass(frozen=True) +class OrderPlan: + """A plan holding the same large entry twice, so the capture shares it.""" + + entries: tuple[OrderPlanEntry, ...] + + +def order_plan() -> OrderPlan: + """Return a plan whose entries repeat one large query and carry a volatile time.""" + + entry: OrderPlanEntry = OrderPlanEntry( + name="order_totals", + query=LONG_QUERY, + invocation_time=datetime(2026, 10, 14, 9, 30, tzinfo=UTC), + data_start=datetime(2026, 4, 1, tzinfo=UTC), + ) + return OrderPlan(entries=(entry, entry)) + + +def expected_order_entry() -> dict[str, object]: + """Return the canonical form of one `order_plan()` entry.""" + + return { + "__type__": f"{__name__}:OrderPlanEntry", + "name": "order_totals", + "query": LONG_QUERY, + "invocation_time": "", + "data_start": {"__datetime__": "2026-04-01T00:00:00+00:00"}, + } diff --git a/tests/unit/scripts/planner_differential/_helpers/capturing/test_canonical.py b/tests/unit/scripts/planner_differential/_helpers/capturing/test_canonical.py new file mode 100644 index 0000000000..b0431ca765 --- /dev/null +++ b/tests/unit/scripts/planner_differential/_helpers/capturing/test_canonical.py @@ -0,0 +1,206 @@ +"""Canonicalisation masks only run-specific values so two runs of one tree compare equal.""" + +from __future__ import annotations + +import pytest + +from scripts.planner_differential._helpers.capturing.canonical import ( + canonical_plan_capture, + canonical_value, + mask_console_text, + mask_text, +) +from scripts.planner_differential._helpers.capturing.state_tables import state_delta +from sqlbuild._native import render_stage_capture +from tests.unit.scripts.planner_differential._helpers.capturing._test_types import ( + CanonicalValueTestCase, + MaskTextTestCase, + PlanCaptureTestCase, + StateDeltaTestCase, +) +from tests.unit.scripts.planner_differential._helpers.capturing.helpers import ( + TEST_MASKS, + expected_order_entry, + order_plan, +) + + +@pytest.mark.parametrize( + "test_case", + [ + MaskTextTestCase( + description="run_id", + raw="run 20261014T002537Z_7e9b74a9fefc", + expected_text="run ", + ), + MaskTextTestCase( + description="project_path_before_work_path", + raw="/runs/work/orders/project/models/a.sql and /runs/work/other", + expected_text="/models/a.sql and /other", + ), + MaskTextTestCase( + description="timestamp_on_the_run_date", + raw="built at 2026-10-14T09:30:00.123+00:00", + expected_text="built at ", + ), + MaskTextTestCase( + description="retention_expiry_after_the_run_date", + raw="expires 2026-11-13 00:38:45.472672", + expected_text="expires ", + ), + MaskTextTestCase( + description="data_timestamp_is_kept", + raw="cursor 2026-04-02 12:00:00 to 2026-04-05 14:30:00", + expected_text="cursor 2026-04-02 12:00:00 to 2026-04-05 14:30:00", + ), + MaskTextTestCase( + description="compact_archive_timestamp", + raw="_sqb_archive__20261014t004426z__migration_origin__orders", + expected_text="_sqb_archive____migration_origin__orders", + ), + ], + ids=lambda case: case.description, +) +def test_given_output_text_when_masking_then_only_run_specific_values_change( + test_case: MaskTextTestCase, +) -> None: + assert mask_text(text=test_case.raw, masks=TEST_MASKS) == test_case.expected_text + + +@pytest.mark.parametrize( + "test_case", + [ + MaskTextTestCase( + description="elapsed_time", + raw="Generated plan. (0.04s)", + expected_text="Generated plan. ()", + ), + MaskTextTestCase( + description="project_path_in_a_diagnostic", + raw="error: /runs/work/orders/project/models/orders.sql:3:1", + expected_text="error: /models/orders.sql:3:1", + ), + ], + ids=lambda case: case.description, +) +def test_given_console_text_when_masking_then_elapsed_times_are_masked( + test_case: MaskTextTestCase, +) -> None: + assert mask_console_text(text=test_case.raw, masks=TEST_MASKS) == test_case.expected_text + + +@pytest.mark.parametrize( + "test_case", + [ + CanonicalValueTestCase( + description="volatile_keys", + value={"run_id": "r1", "ts": "t", "migration_event_id": "e", "name": "orders"}, + expected_value={ + "run_id": "", + "ts": "", + "migration_event_id": "", + "name": "orders", + }, + ), + CanonicalValueTestCase( + description="absent_volatile_value_stays_null", + value={"completed_at": None}, + expected_value={"completed_at": None}, + ), + CanonicalValueTestCase( + description="non_string_key_mapping_is_sorted", + value={"__mapping__": [[["model", "orders"], 2], [["loader", "customers"], 1]]}, + expected_value={ + "__mapping__": [[["loader", "customers"], 1], [["model", "orders"], 2]] + }, + ), + CanonicalValueTestCase( + description="list_order_is_kept", + value={"execution_order": ["orders", "customers"]}, + expected_value={"execution_order": ["orders", "customers"]}, + ), + CanonicalValueTestCase( + description="nested_strings_are_masked", + value=[{"path": "/runs/work/orders/project/models/orders.sql"}], + expected_value=[{"path": "/models/orders.sql"}], + ), + ], + ids=lambda case: case.description, +) +def test_given_decoded_json_when_canonicalising_then_volatile_values_are_masked( + test_case: CanonicalValueTestCase, +) -> None: + assert canonical_value(value=test_case.value, masks=TEST_MASKS) == test_case.expected_value + + +@pytest.mark.parametrize( + "test_case", + [ + StateDeltaTestCase( + description="appended_rows_are_added", + before={"db:main._sqlbuild_fingerprints": ['{"name": "orders"}']}, + after={"db:main._sqlbuild_fingerprints": ['{"name": "orders"}', '{"name": "orders"}']}, + expected_delta={ + "db:main._sqlbuild_fingerprints": {"added": [{"name": "orders"}], "removed": []} + }, + ), + StateDeltaTestCase( + description="replaced_and_new_tables", + before={"db:main._sqlbuild_migrations": ['{"state": "old"}']}, + after={ + "db:main._sqlbuild_migrations": ['{"state": "new"}'], + "db:main._sqlbuild_old_name_views": ['{"view": "orders"}'], + }, + expected_delta={ + "db:main._sqlbuild_migrations": { + "added": [{"state": "new"}], + "removed": [{"state": "old"}], + }, + "db:main._sqlbuild_old_name_views": { + "added": [{"view": "orders"}], + "removed": [], + }, + }, + ), + StateDeltaTestCase( + description="unchanged_tables_are_omitted", + before={"db:main._sqlbuild_fingerprints": ['{"name": "orders"}']}, + after={"db:main._sqlbuild_fingerprints": ['{"name": "orders"}']}, + expected_delta={}, + ), + ], + ids=lambda case: case.description, +) +def test_given_state_rows_when_diffing_then_gained_and_lost_rows_are_reported( + test_case: StateDeltaTestCase, +) -> None: + assert state_delta(before=test_case.before, after=test_case.after) == test_case.expected_delta + + +@pytest.mark.parametrize( + "test_case", + [ + PlanCaptureTestCase( + description="shared_entries_expanded_and_invocation_time_masked", + value=order_plan(), + expected_value={ + "__type__": ( + "tests.unit.scripts.planner_differential._helpers.capturing.helpers:OrderPlan" + ), + "entries": [expected_order_entry(), expected_order_entry()], + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_plan_capture_document_when_canonicalising_then_nodes_expand_and_mask( + test_case: PlanCaptureTestCase, +) -> None: + assert ( + canonical_plan_capture(text=render_stage_capture(test_case.value), masks=TEST_MASKS) + == test_case.expected_value + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/src/sqlbuild/sql_values/main/__init__.py b/tests/unit/scripts/planner_differential/_helpers/steps/__init__.py similarity index 100% rename from src/sqlbuild/sql_values/main/__init__.py rename to tests/unit/scripts/planner_differential/_helpers/steps/__init__.py diff --git a/tests/unit/scripts/planner_differential/_helpers/steps/_test_types.py b/tests/unit/scripts/planner_differential/_helpers/steps/_test_types.py new file mode 100644 index 0000000000..d2b5436449 --- /dev/null +++ b/tests/unit/scripts/planner_differential/_helpers/steps/_test_types.py @@ -0,0 +1,35 @@ +"""Test case types for the planner differential model-file edits.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from scripts.planner_differential.models import ModelFile + + +@dataclass(frozen=True) +class SplitModelFileTestCase: + """Model file text and the header and query the harness edits, or None.""" + + description: str + text: str + expected_model_file: ModelFile | None + + +@dataclass(frozen=True) +class HeaderPropertyTestCase: + """A MODEL header and the header after inserting one property line.""" + + description: str + header: str + line: str + expected_header: str + + +@dataclass(frozen=True) +class JoinModelFileTestCase: + """A split model file and the text the harness writes back.""" + + description: str + model_file: ModelFile + expected_text: str diff --git a/tests/unit/scripts/planner_differential/_helpers/steps/test_model_file.py b/tests/unit/scripts/planner_differential/_helpers/steps/test_model_file.py new file mode 100644 index 0000000000..7832794c03 --- /dev/null +++ b/tests/unit/scripts/planner_differential/_helpers/steps/test_model_file.py @@ -0,0 +1,108 @@ +"""Model-file edits split the MODEL header from a single query and insert properties.""" + +from __future__ import annotations + +import pytest + +from scripts.planner_differential._helpers.steps.model_file import ( + add_header_property, + join_model_file, + split_model_file, +) +from scripts.planner_differential.models import ModelFile +from tests.unit.scripts.planner_differential._helpers.steps._test_types import ( + HeaderPropertyTestCase, + JoinModelFileTestCase, + SplitModelFileTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SplitModelFileTestCase( + description="multi_line_header", + text="MODEL (\n materialized table,\n columns (id (nullable false)),\n);\n\n" + 'SELECT id FROM __ref("orders");\n', + expected_model_file=ModelFile( + header="MODEL (\n materialized table,\n columns (id (nullable false)),\n);", + query='SELECT id FROM __ref("orders")', + ), + ), + SplitModelFileTestCase( + description="inline_header", + text="MODEL (materialized view);\nSELECT 1 AS id\n", + expected_model_file=ModelFile( + header="MODEL (materialized view);", query="SELECT 1 AS id" + ), + ), + SplitModelFileTestCase( + description="no_header", + text="SELECT 1 AS id\n", + expected_model_file=ModelFile(header="", query="SELECT 1 AS id"), + ), + SplitModelFileTestCase( + description="several_statements_are_not_edited", + text="MODEL (\n materialized table,\n);\n\nSET x = 1;\nSELECT 1 AS id\n", + expected_model_file=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_model_file_when_splitting_then_header_and_query_are_separated( + test_case: SplitModelFileTestCase, +) -> None: + assert split_model_file(test_case.text) == test_case.expected_model_file + + +@pytest.mark.parametrize( + "test_case", + [ + HeaderPropertyTestCase( + description="existing_header", + header="MODEL (\n materialized table,\n);", + line="tags [orders],", + expected_header="MODEL (\n tags [orders],\n materialized table,\n);", + ), + HeaderPropertyTestCase( + description="no_header", + header="", + line="tags [orders],", + expected_header="MODEL (\n tags [orders],\n);", + ), + ], + ids=lambda case: case.description, +) +def test_given_header_when_adding_property_then_it_leads_the_header( + test_case: HeaderPropertyTestCase, +) -> None: + assert ( + add_header_property(header=test_case.header, line=test_case.line) + == test_case.expected_header + ) + + +@pytest.mark.parametrize( + "test_case", + [ + JoinModelFileTestCase( + description="header_and_query", + model_file=ModelFile(header="MODEL (\n materialized view,\n);", query="SELECT 1"), + expected_text="MODEL (\n materialized view,\n);\n\nSELECT 1\n", + ), + JoinModelFileTestCase( + description="query_only", + model_file=ModelFile(header="", query="SELECT 1"), + expected_text="SELECT 1\n", + ), + ], + ids=lambda case: case.description, +) +def test_given_split_file_when_joining_then_header_and_query_are_separated( + test_case: JoinModelFileTestCase, +) -> None: + assert join_model_file(test_case.model_file) == test_case.expected_text + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/helpers.py b/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/helpers.py index 0a29327e9d..bb8a7c17b8 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/helpers.py +++ b/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/helpers.py @@ -1,6 +1,6 @@ from __future__ import annotations -from sqlbuild.sql_values.main.normalize import normalize_sql_value +from sqlbuild._native import normalize_sql_value from sqlbuild.sql_values.models import SqlValue diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py index a3207baee3..f310432175 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py @@ -87,7 +87,7 @@ def test_given_pending_run_when_showing_detail_then_persisted_summary_is_refresh lambda **kwargs: SimpleNamespace(project_config=object(), local_config=object()), ) monkeypatch.setattr( - "sqlbuild.spec.contracts.main.resolve_effective_adapter_name.resolve_effective_adapter_name", + "sqlbuild._native.resolve_effective_adapter_name", lambda **kwargs: "snowflake", ) monkeypatch.setattr( @@ -99,7 +99,7 @@ def test_given_pending_run_when_showing_detail_then_persisted_summary_is_refresh lambda **kwargs: {"database": "ANALYTICS"}, ) monkeypatch.setattr( - "sqlbuild.compiler.compile.main._effective_target.build_effective_target_config", + "sqlbuild._native.build_effective_target_config", lambda **kwargs: SimpleNamespace(database=None), ) request: CostCommandRequest = CostCommandRequest( @@ -191,7 +191,7 @@ def test_given_useful_partial_when_refresh_degrades_then_persisted_summary_is_pr lambda **kwargs: SimpleNamespace(project_config=object(), local_config=object()), ) monkeypatch.setattr( - "sqlbuild.spec.contracts.main.resolve_effective_adapter_name.resolve_effective_adapter_name", + "sqlbuild._native.resolve_effective_adapter_name", lambda **kwargs: "snowflake", ) monkeypatch.setattr( @@ -274,7 +274,7 @@ def test_given_persisted_ledger_failure_when_refresh_has_no_coverage_then_failur lambda **kwargs: SimpleNamespace(project_config=object(), local_config=object()), ) monkeypatch.setattr( - "sqlbuild.spec.contracts.main.resolve_effective_adapter_name.resolve_effective_adapter_name", + "sqlbuild._native.resolve_effective_adapter_name", lambda **kwargs: "snowflake", ) monkeypatch.setattr( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py index 7f8d8e2281..fbc50afd1e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py @@ -4,9 +4,7 @@ from dataclasses import replace -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup -from sqlbuild.compiler.scopes.main.preview_scope_move import preview_scope_move -from sqlbuild.compiler.scopes.main.query_scope_report import query_scope_report +from sqlbuild._native import build_scope_lookup, preview_scope_move, query_scope_report from sqlbuild.compiler.scopes.models import ( OwnershipRoot, ResourceIdentity, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index c48c7dfd15..0c85d6fe57 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -1,5 +1,6 @@ from __future__ import annotations +import io from collections.abc import Callable from pathlib import Path from typing import Any @@ -63,6 +64,13 @@ ) +class TtyStdout(io.StringIO): + """A captured stdout that reports a terminal.""" + + def isatty(self) -> bool: + return True + + class NoConnectDuckDbAdapter(DuckDbAdapter): """DuckDB adapter test double that fails if compile opens a connection.""" diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py index 4544a713e1..34f8c5f0c9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py @@ -5,12 +5,10 @@ import pytest +from sqlbuild._native import resolve_effective_adapter_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( ExpectedMessageTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 4572ca2eca..4b6351bb72 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -19,7 +19,6 @@ from sqlbuild.compiler.planner.main.execution import ( _plan_compiled_sql_test_artifacts as plan_artifacts_module, ) -from sqlbuild.presentation.classes import transient_status_reporter from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileColumnContractModeTestCase, CompileCommandTestCase, @@ -32,6 +31,7 @@ ) from tests.unit.src.sqlbuild.cli.commands.main.compile.helpers import ( NoConnectDuckDbAdapter, + TtyStdout, prepare_python_compile_project, prepare_rule_gated_compile_project, prepare_static_compile_project, @@ -101,38 +101,9 @@ def test_given_tty_stdout_when_running_compile_then_it_persists_phase_timings( monkeypatch: pytest.MonkeyPatch, ) -> None: project_dir: Path = prepare_static_compile_project(tmp_path) - started_messages: list[str] = [] - completed_messages: list[str] = [] - - class FakeStdout: - def isatty(self) -> bool: - return True - - def write(self, text: str) -> int: - return len(text) - - def flush(self) -> None: - return None - - class FakeStatusReporter: - def __init__(self, **kwargs: object) -> None: - del kwargs - - def start(self, message: str) -> None: - started_messages.append(message) - - def update(self, message: str) -> None: - started_messages.append(message) - - def complete(self, message: str, *, blank_line_after: bool = False) -> None: - del blank_line_after - completed_messages.append(message) - - def close(self) -> None: - return None - - monkeypatch.setattr(sys, "stdout", FakeStdout()) - monkeypatch.setattr(transient_status_reporter, "TransientStatusReporter", FakeStatusReporter) + stdout: TtyStdout = TtyStdout() + monkeypatch.setenv("SQLBUILD_NO_PROGRESS", "1") + monkeypatch.setattr(sys, "stdout", stdout) monkeypatch.setattr( runtime_adapters, "resolve_adapter", @@ -140,21 +111,22 @@ def close(self) -> None: ) exit_code: int = run_compile( - CompileCommandRequest(project_dir=project_dir, no_sql_validation=True) + CompileCommandRequest(project_dir=project_dir, no_sql_validation=True, no_color=True) ) + rendered_stdout: str = stdout.getvalue() assert exit_code == test_case.expected_exit_code - assert started_messages == [ + for message in ( "Discovering project...", "Compiling project graph...", "Analyzing column lineage...", "Validating model contracts...", "Evaluating built-in and custom rules...", "Writing compiled artifacts...", - ] - assert len(completed_messages) == len(test_case.expected_stdout_fragments) - for index, fragment in enumerate(test_case.expected_stdout_fragments): - assert completed_messages[index].startswith(f"{fragment} (") + ): + assert f"{message}\n" in rendered_stdout + for fragment in test_case.expected_stdout_fragments: + assert f"\n{fragment} (" in rendered_stdout @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 1c3fb5caf6..047dd8951e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -9,6 +9,7 @@ import sqlbuild._native as _native from sqlbuild._native import ( + build_manifest, compiled_model_record, project_with, write_compile_target, @@ -25,7 +26,6 @@ CompiledResourceType, DiagnosticPhase, ) -from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.planner.main.execution import ( _plan_compiled_sql_test_artifacts as plan_artifacts_module, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py index 131bee4ac8..f5a4a72bee 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py @@ -6,12 +6,11 @@ from pathlib import Path from typing import Any, Protocol, cast -from sqlbuild._native import NativeProgressProjector +from sqlbuild._native import NativeProgressProjector, loader_destination_parts from sqlbuild.cli.entry.models import ( CliEntrypointHandlers, ) from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.main.loader_destination_parts import loader_destination_parts class JsonOutputRequest(Protocol): diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py index 4be4a9b530..1c03c3f7a8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py @@ -1,34 +1,12 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.auditing.types import AuditOutcome from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.runtime.contracts.types import ExecutionResourceKind -@dataclass(frozen=True) -class TruncateNameTestCase: - description: str - name: str - width: int - expected_result: str - - -@dataclass(frozen=True) -class AuditAggregationTestCase: - description: str - audit_results: tuple[AuditExecutionResult, ...] - expected_entry_count: int - expected_labels: tuple[str, ...] - expected_outcomes: tuple[AuditOutcome, ...] - expected_batch_totals: tuple[int, ...] - expected_batch_passes: tuple[int, ...] - expected_reused: tuple[bool, ...] = field(default_factory=tuple) - - @dataclass(frozen=True) class BuildFooterTestCase: description: str diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py index fd42c3c6c9..39e0dbcb10 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py @@ -3,8 +3,6 @@ from __future__ import annotations import re -import threading -from collections.abc import Callable from pathlib import Path from sqlbuild.compiler.auditing.types import ( @@ -32,15 +30,6 @@ ) -def write_spinner_line_and_release( - *, - write_spinner_line: Callable[[], None], - spinner_updates: threading.Semaphore, -) -> None: - write_spinner_line() - spinner_updates.release() - - def build_audit_result( *, name: str, diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py index 435f92a3c7..0be484741e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py @@ -2,32 +2,24 @@ from __future__ import annotations -import threading -from collections.abc import Callable +import time from functools import partial from io import StringIO import pytest from sqlbuild._native import ( + BuildProgressCallbacks, + NestedCommandProgressCallbacks, build_test_expectation_rows, + format_build_footer, resolve_test_name_width, write_execution_header, ) from sqlbuild.adapter.contract.models import LifeCycleEvent from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.commands.classes import build_progress_callbacks -from sqlbuild.cli.commands.classes.build_progress_callbacks import ( - BuildProgressCallbacks, - _aggregate_audit_results, - _truncate_name, - format_build_footer, -) -from sqlbuild.cli.progress.classes.nested_command_progress_callbacks import ( - NestedCommandProgressCallbacks, -) -from sqlbuild.cli.progress.models import AuditDisplayEntry, NestedProgressChildRow -from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope +from sqlbuild.cli.progress.models import NestedProgressChildRow +from sqlbuild.compiler.auditing.types import AuditOutcome from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cost.models import StatementExecutionTelemetry @@ -44,9 +36,9 @@ from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.executor.testing.types import SqlTestOutcome +from sqlbuild.presentation.main import terminal_columns as terminal_columns_module from sqlbuild.runtime.contracts.types import ExecutionResourceKind from tests.unit.src.sqlbuild.cli.commands.shared._helpers._test_types import ( - AuditAggregationTestCase, BuildFooterTestCase, BuildProgressActiveSpinnerTestCase, BuildProgressFailureOutputTestCase, @@ -60,7 +52,6 @@ NestedProgressConcurrentTestCase, RuntimeDiagnosticsTestCase, SqlTestStatusAlignmentTestCase, - TruncateNameTestCase, ) from tests.unit.src.sqlbuild.cli.commands.shared._helpers.helpers import ( build_audit_result, @@ -68,7 +59,6 @@ build_sql_test_plan_entry, build_sql_test_result, status_columns, - write_spinner_line_and_release, ) @@ -201,44 +191,6 @@ def test_given_terminal_scheduler_state_when_reporting_then_output_explains_abor assert absent_fragment not in output -@pytest.mark.parametrize( - "test_case", - [ - TruncateNameTestCase( - description="name shorter than width is returned unchanged", - name="orders", - width=20, - expected_result="orders", - ), - TruncateNameTestCase( - description="name exactly at width is returned unchanged", - name="orders", - width=6, - expected_result="orders", - ), - TruncateNameTestCase( - description="name longer than width is truncated with ellipsis", - name="very_long_model_name_that_exceeds_limit", - width=20, - expected_result="very_long_model_n...", - ), - TruncateNameTestCase( - description="name one char over width truncates correctly", - name="abcdefgh", - width=7, - expected_result="abcd...", - ), - ], - ids=lambda case: case.description, -) -def test_given_name_and_width_when_truncating_then_returns_expected_result( - test_case: TruncateNameTestCase, -) -> None: - result: str = _truncate_name(name=test_case.name, width=test_case.width) - - assert result == test_case.expected_result - - @pytest.mark.parametrize( "test_case", [ @@ -352,194 +304,6 @@ def test_given_overlapping_items_when_completing_out_of_order_then_headers_follo assert stream.getvalue() == test_case.expected_output -@pytest.mark.parametrize( - "test_case", - [ - AuditAggregationTestCase( - description="final-only audits produce one entry per audit with audit label", - audit_results=( - build_audit_result(name="not_null", outcome=AuditOutcome.PASS, column_name="id"), - build_audit_result(name="unique", outcome=AuditOutcome.PASS, column_name="id"), - ), - expected_entry_count=2, - expected_labels=("audit", "audit"), - expected_outcomes=(AuditOutcome.PASS, AuditOutcome.PASS), - expected_batch_totals=(1, 1), - expected_batch_passes=(1, 1), - expected_reused=(False, False), - ), - AuditAggregationTestCase( - description="reused final audit records reused display flag", - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - reused=True, - ), - ), - expected_entry_count=1, - expected_labels=("audit",), - expected_outcomes=(AuditOutcome.PASS,), - expected_batch_totals=(1,), - expected_batch_passes=(1,), - expected_reused=(True,), - ), - AuditAggregationTestCase( - description="delta_and_final audits produce separate delta and final entries", - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.FINAL, - ), - ), - expected_entry_count=2, - expected_labels=("audit (d)", "audit (f)"), - expected_outcomes=(AuditOutcome.PASS, AuditOutcome.PASS), - expected_batch_totals=(1, 1), - expected_batch_passes=(1, 1), - expected_reused=(False, False), - ), - AuditAggregationTestCase( - description="multiple delta batches aggregate into one entry with batch count", - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.FINAL, - ), - ), - expected_entry_count=2, - expected_labels=("audit (d)", "audit (f)"), - expected_outcomes=(AuditOutcome.PASS, AuditOutcome.PASS), - expected_batch_totals=(3, 1), - expected_batch_passes=(3, 1), - expected_reused=(False, False), - ), - AuditAggregationTestCase( - description="worst outcome across batches is reported when one batch fails", - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.ERROR, - column_name="id", - row_count=5, - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.FINAL, - ), - ), - expected_entry_count=2, - expected_labels=("audit (d)", "audit (f)"), - expected_outcomes=(AuditOutcome.ERROR, AuditOutcome.PASS), - expected_batch_totals=(3, 1), - expected_batch_passes=(2, 1), - expected_reused=(False, False), - ), - AuditAggregationTestCase( - description="warn is worst outcome when no errors present", - audit_results=( - build_audit_result( - name="row_check", - outcome=AuditOutcome.PASS, - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="row_check", - outcome=AuditOutcome.WARN, - row_count=2, - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="row_check", - outcome=AuditOutcome.PASS, - run_scope_phase=AuditRunScope.FINAL, - ), - ), - expected_entry_count=2, - expected_labels=("audit (d)", "audit (f)"), - expected_outcomes=(AuditOutcome.WARN, AuditOutcome.PASS), - expected_batch_totals=(2, 1), - expected_batch_passes=(1, 1), - expected_reused=(False, False), - ), - ], - ids=lambda case: case.description, -) -def test_given_audit_results_when_aggregating_then_produces_expected_entries( - test_case: AuditAggregationTestCase, -) -> None: - entries: list[AuditDisplayEntry] = _aggregate_audit_results(test_case.audit_results) - - assert len(entries) == test_case.expected_entry_count - - idx: int - for idx in range(test_case.expected_entry_count): - assert entries[idx].label == test_case.expected_labels[idx], ( - f"entry {idx}: expected label {test_case.expected_labels[idx]!r}, " - f"got {entries[idx].label!r}" - ) - assert entries[idx].outcome == test_case.expected_outcomes[idx], ( - f"entry {idx}: expected outcome {test_case.expected_outcomes[idx]!r}, " - f"got {entries[idx].outcome!r}" - ) - assert entries[idx].batch_total == test_case.expected_batch_totals[idx], ( - f"entry {idx}: expected batch_total {test_case.expected_batch_totals[idx]}, " - f"got {entries[idx].batch_total}" - ) - assert entries[idx].batch_pass == test_case.expected_batch_passes[idx], ( - f"entry {idx}: expected batch_pass {test_case.expected_batch_passes[idx]}, " - f"got {entries[idx].batch_pass}" - ) - assert entries[idx].reused == test_case.expected_reused[idx], ( - f"entry {idx}: expected reused {test_case.expected_reused[idx]}, " - f"got {entries[idx].reused}" - ) - - @pytest.mark.parametrize( "test_case", [ @@ -932,7 +696,7 @@ def test_given_failed_top_level_node_when_reporting_progress_then_writes_error_d stream: StringIO = StringIO() monkeypatch.setattr("sys.stdout", stream) monkeypatch.setattr( - build_progress_callbacks, + terminal_columns_module, "terminal_columns", partial(int, test_case.terminal_width), ) @@ -1292,23 +1056,10 @@ def test_given_active_top_level_node_when_waiting_then_spinner_advances_frames( plan=PlanOutput(), use_color=False, ) - spinner_updates: threading.Semaphore = threading.Semaphore(value=0) - write_spinner_line: Callable[[], None] = callbacks._write_spinner_line - monkeypatch.setattr( - callbacks, - "_write_spinner_line", - partial( - write_spinner_line_and_release, - write_spinner_line=write_spinner_line, - spinner_updates=spinner_updates, - ), - ) callbacks.on_node_start(name=test_case.node_name, resource_kind=test_case.node_type) try: - _spinner_frame: str - for _spinner_frame in test_case.expected_spinner_frames: - assert spinner_updates.acquire(timeout=test_case.timeout_seconds) + time.sleep(test_case.timeout_seconds) finally: callbacks.on_node_complete( ModelExecutionResult( diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py index cd7a18324b..fbe30bb55e 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py @@ -14,9 +14,7 @@ ) from tests.unit.src.sqlbuild.cli.compile_reuse.main.helpers import compile_in_fresh_process -_FULL_COMPILE_MODULES: tuple[str, ...] = ( - "sqlbuild.compiler.compile._helpers.native_stages.assembly", -) +_FULL_COMPILE_MODULES: tuple[str, ...] = ("sqlbuild.compiler.analysis_session.models",) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py b/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py index 7cd228ed06..35846a539d 100644 --- a/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py +++ b/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py @@ -8,11 +8,11 @@ import pytest from sqlbuild._native import ( + BuildProgressCallbacks, NativeProgressProjector, TerminalEventIndex, terminal_event_index_scope, ) -from sqlbuild.cli.commands.classes.build_progress_callbacks import BuildProgressCallbacks from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 972ab91505..230ed782a7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -45,16 +45,6 @@ class SqlReferenceExtractionErrorTestCase: expected_help: str | None = None -@dataclass(frozen=True) -class StaticProjectVarBatchTestCase: - """Model SQL texts and the `(sql, error)` each interpolates to.""" - - description: str - sqls: tuple[str, ...] - effective_vars: dict[str, object] - expected_results: tuple[tuple[str, str | None], ...] - - @dataclass(frozen=True) class AuditFactoryAttachmentTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py index 0fbec2669a..15e844b23e 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py @@ -8,8 +8,8 @@ build_compile_inputs, discover_project_inputs, native_model_config_session, + record_template_reads, ) -from sqlbuild.compiler.compile._helpers.render.templating import record_template_reads from sqlbuild.compiler.compile.models import ( CompileModelConfig, NativeModelConfigInputs, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 8bae1a3e59..2a730a73fc 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -8,6 +8,9 @@ from sqlbuild._native import ( assemble_project, build_compile_inputs, + build_manifest, + build_native_declaration_contexts, + build_scope_lookup, discover_project_inputs, expand_sql_macros, infer_native_query_columns, @@ -59,13 +62,8 @@ ColumnTransformKind, InferredNullability, ) -from sqlbuild.compiler.manifest.main.build import build_manifest -from sqlbuild.compiler.model_loop.main._build_native_declaration_contexts import ( - build_native_declaration_contexts, -) from sqlbuild.compiler.planner._helpers.graph.core import build_execution_upstream_deps from sqlbuild.compiler.planner._helpers.resolve.refs import resolve_ref_references -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py index 8520ec0c33..6fd7f9f166 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py @@ -2,14 +2,14 @@ import pytest -from sqlbuild._native import infer_native_query_columns -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery -from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( +from sqlbuild._native import ( cursor_intrinsics_analysis_sql, - get_validated_model_cursor_intrinsics, - render_cursor_intrinsics, + infer_native_query_columns, + resolve_cursor_intrinsics, + validated_model_cursor_intrinsics, ) +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.analysis_session.models import NativeColumnQuery from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.lineage.types import InferredNullability @@ -35,7 +35,7 @@ def test_given_valid_incremental_query_when_validating_then_canonicalizes_intrinsics( test_case: CursorIntrinsicValidationTestCase, ) -> None: - sql: str = get_validated_model_cursor_intrinsics( + sql: str = validated_model_cursor_intrinsics( sql=test_case.sql, config_values={"materialized": "incremental", "cursor": "event_time"}, model_name="events", @@ -72,7 +72,7 @@ def test_given_valid_incremental_query_when_validating_then_canonicalizes_intrin def test_given_comments_and_literals_when_rendering_then_only_replaces_executable_calls( test_case: CursorIntrinsicRenderTestCase, ) -> None: - sql: str = render_cursor_intrinsics( + sql, _ = resolve_cursor_intrinsics( sql=test_case.sql, start_sql="TIMESTAMP '2026-01-01'", end_sql="TIMESTAMP '2026-01-02'", @@ -106,7 +106,7 @@ def test_given_invalid_intrinsic_when_validating_then_raises( test_case: CursorIntrinsicErrorTestCase, ) -> None: with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - get_validated_model_cursor_intrinsics( + validated_model_cursor_intrinsics( sql=test_case.sql, config_values={"materialized": "incremental", "cursor": "event_time"}, model_name="events", diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py index 9d6cdffbaf..541ed2451e 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py @@ -9,7 +9,12 @@ import pytest -from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs +from sqlbuild._native import ( + assemble_project, + build_compile_inputs, + discover_project_inputs, + scope_metadata_projection, +) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView, ModelInput from sqlbuild.compiler.discovery.models import ( @@ -17,7 +22,6 @@ DiscoveredSourceFile, SqlHookEntry, ) -from sqlbuild.compiler.scopes.main.scope_metadata import scope_metadata_projection from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, ResourceIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py index 4796b9a35e..88f9f0f96c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild.compiler.compile._helpers.render.spans import map_through_passes -from sqlbuild.compiler.compile._helpers.render.sql_vars import expand_authored_sql_with_spans +from sqlbuild._native import expand_authored_sql_with_spans, map_expanded_offset from sqlbuild.compiler.compile.models import ExpansionSpan, LoadedMacro, MacroContext, MappedOffset from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ExpandWithSpansTestCase from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( @@ -98,7 +97,7 @@ def test_given_expanded_offset_when_mapping_then_authored_position_matches( collection_rendering=DUCKDB_COMPILE_ADAPTER_CONTEXT.collection_rendering, ) assert expanded_sql == test_case.expected_expanded_sql - resolved: MappedOffset = map_through_passes( + resolved: MappedOffset = map_expanded_offset( offset=test_case.probe_expanded_offset, passes=passes ) assert resolved.offset == test_case.expected_authored_offset diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py index a7a4ad428d..f07166bac9 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py @@ -4,9 +4,7 @@ import pytest -from sqlbuild.compiler.compile.main._analyze_columns_and_lineage import ( - analyze_resolved_column_reads, -) +from sqlbuild._native import analyze_resolved_column_reads from sqlbuild.compiler.compile.models import CompiledLineageSourceFact, CompileSqlReference from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.references.types import SqlReferenceKind diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_spans.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_spans.py index 29ad65be0c..359f824121 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_spans.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_spans.py @@ -4,10 +4,7 @@ import pytest -from sqlbuild.compiler.compile._helpers.render.spans import ( - map_output_offset, - map_through_passes, -) +from sqlbuild._native import map_expanded_offset from sqlbuild.compiler.compile.models import ExpansionSpan, MappedOffset from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( MapOutputOffsetTestCase, @@ -76,7 +73,7 @@ def test_given_rendered_offset_when_mapping_one_pass_then_source_offset_matches( test_case: MapOutputOffsetTestCase, ) -> None: - resolved: MappedOffset = map_output_offset(offset=test_case.offset, spans=test_case.spans) + resolved: MappedOffset = map_expanded_offset(offset=test_case.offset, passes=(test_case.spans,)) assert resolved.offset == test_case.expected_offset assert resolved.generated == test_case.expected_generated @@ -137,6 +134,6 @@ def test_given_rendered_offset_when_mapping_one_pass_then_source_offset_matches( def test_given_rendered_offset_when_mapping_all_passes_then_authored_offset_matches( test_case: MapThroughPassesTestCase, ) -> None: - resolved: MappedOffset = map_through_passes(offset=test_case.offset, passes=test_case.passes) + resolved: MappedOffset = map_expanded_offset(offset=test_case.offset, passes=test_case.passes) assert resolved.offset == test_case.expected_offset assert resolved.generated == test_case.expected_generated diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py index d729ea72fb..07f3824c8f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py @@ -2,12 +2,9 @@ import pytest -from sqlbuild._native import infer_native_query_columns +from sqlbuild._native import infer_native_query_columns, substitute_placeholder_defaults from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns -from sqlbuild.compiler.compile._helpers.analysis.reference_names import ( - substitute_placeholder_defaults, -) from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, CompiledLineageSourceFact, @@ -944,7 +941,7 @@ def test_given_sql_with_placeholders_when_substituting_then_returns_expected( test_case: SubstitutePlaceholderDefaultsTestCase, ) -> None: result: str = substitute_placeholder_defaults( - query_sql=test_case.query_sql, placeholders=test_case.placeholders + test_case.query_sql, list(test_case.placeholders.items()) ) assert result == test_case.expected_sql diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py index 39898fea7e..d070fba082 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py @@ -6,14 +6,9 @@ import pytest -from sqlbuild.compiler.compile._helpers.render.sql_vars import ( - interpolate_sql_batch, - substitute_sql_vars_with_spans, -) +from sqlbuild._native import substitute_sql_vars_with_spans from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import SqlInterpolation from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( - StaticProjectVarBatchTestCase, SubstituteSqlVarsErrorTestCase, SubstituteSqlVarsTestCase, ) @@ -24,6 +19,18 @@ @pytest.mark.parametrize( "test_case", [ + SubstituteSqlVarsTestCase( + description="skips dollar-quoted text and positional parameters", + sql="SELECT $tag$ it's $$ -- @@status $tag$, price$1$, $1, @@revision", + effective_vars={"revision": 7, "status": "ready"}, + expected_sql="SELECT $tag$ it's $$ -- ready $tag$, price$1$, $1, 7", + ), + SubstituteSqlVarsTestCase( + description="keeps deferred placeholders and commented tokens", + sql="-- @@revision\nSELECT @@revision, '@@status', @@@window_start", + effective_vars={"revision": 7, "status": "ready"}, + expected_sql="-- @@revision\nSELECT 7, 'ready', @@@window_start", + ), SubstituteSqlVarsTestCase( description="replaces project var in SQL expression", sql="SELECT * FROM orders WHERE region = @@target_region", @@ -223,48 +230,5 @@ def test_given_missing_var_when_substituting_then_raises( ) -@pytest.mark.parametrize( - "test_case", - [ - StaticProjectVarBatchTestCase( - description="each model is interpolated; an error waits in its own result", - sqls=( - "SELECT @@revision, '@@status', @@@window_start", - "-- @@revision\nSELECT 1", - "SELECT '@@ENV:SQLBUILD_TEST_BATCH_UNSET'", - "SELECT $tag$ it's $$ -- @@status $tag$, price$1$, $1, @@revision", - "SELECT @@grants", - ), - effective_vars={"revision": 7, "status": "ready", "grants": {"role": "analyst"}}, - expected_results=( - ("SELECT 7, 'ready', @@@window_start", None), - ("-- @@revision\nSELECT 1", None), - ( - "SELECT '@@ENV:SQLBUILD_TEST_BATCH_UNSET'", - "unknown environment variable '@@ENV:SQLBUILD_TEST_BATCH_UNSET' in " - "'models/test_model.sql'", - ), - ("SELECT $tag$ it's $$ -- ready $tag$, price$1$, $1, 7", None), - ( - "SELECT @@grants", - "SQL variable '@@grants' is an object and cannot be interpolated as text: " - '{"role":"analyst"}. Use a macro to consume structured vars.', - ), - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_model_sqls_when_interpolating_a_batch_then_each_holds_its_sql_or_error( - test_case: StaticProjectVarBatchTestCase, -) -> None: - results: tuple[SqlInterpolation, ...] = interpolate_sql_batch( - sqls=tuple((sql, _FILE_PATH) for sql in test_case.sqls), - effective_vars=test_case.effective_vars, - ) - - assert [(result.sql, result.error) for result in results] == list(test_case.expected_results) - - if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py index 90e9062385..b418fe8604 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.compile._helpers.render.templating import ( +from sqlbuild._native import ( expand_effective_vars, expand_template_data, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py b/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py index e14cc9e776..6617c86aa4 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild.compiler.compile._helpers.render.context_templates import resolve_run_id -from sqlbuild.compiler.compile.main.effective_runtime import build_effective_runtime_config +from sqlbuild._native import build_effective_runtime_config, resolve_run_id from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.models import ( LocalConfig, diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_main.py b/tests/unit/src/sqlbuild/compiler/compile/test_main.py index fe89344a7f..61850194b7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_main.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild._native import build_compile_inputs, discover_project_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs, resolve_target_config from sqlbuild.compiler.compile.models import ( CompileModelConfig, ) @@ -19,7 +19,6 @@ ) from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.models import ( AuthoredTimeTravelRetention, ClonePolicy, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py index d427cd28b2..24bc174f74 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py @@ -6,11 +6,8 @@ import pytest -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import discover_project_inputs, validate_resource_identity from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError -from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( - validate_resource_identity, -) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( CanonicalResourceIdentityTestCase, DiscoveredResourceIdentityErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_diff.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_diff.py index d0f29c74d3..b8ce19c597 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_diff.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_diff.py @@ -6,8 +6,8 @@ import pytest +from sqlbuild._native import resolve_target_config from sqlbuild.compiler.discovery.exceptions import ProjectConfigError -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( TargetDiffConfigErrorTestCase, TargetDiffConfigTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py index 9dfcd8b46e..86c2bba61f 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import build_manifest from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditRunScope, @@ -34,7 +35,6 @@ DiscoveredSourceFile, DiscoveredSqlHookFile, ) -from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.planner.models import ( AuditPlanEntry, ChainStep, diff --git a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py index bbacfc9788..9cf42aeebf 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py @@ -9,6 +9,7 @@ import sqlbuild._native as _native from sqlbuild._native import ( + build_manifest, project_with, ) from sqlbuild.compiler.compile.models import ( @@ -22,7 +23,6 @@ CompiledSourceView, ) from sqlbuild.compiler.discovery.models import DiscoveredSqlHookFile, SqlHookEntry -from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.planner.models import AuditPlanEntry, ChainStep, PlanOutput, SqlTestPlanEntry from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py index 7ddaecfa59..b83e6509d0 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py @@ -2,9 +2,9 @@ import pytest +from sqlbuild._native import run_clone_pipeline_with_options from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.clone import prepare_clone_pipeline from sqlbuild.compiler.pipeline.models import ( ClonePipelineConnection, ClonePipelineOptions, @@ -54,7 +54,7 @@ def test_given_clone_target_without_schema_when_compiling_then_fails_before_plan ValueError, match=f"Named target '{test_case.expected_target_name}' must explicitly set schema", ): - prepare_clone_pipeline( + run_clone_pipeline_with_options( discovered_inputs=discovered_inputs, adapter=PostgresAdapter(), origin_target_name="prod", @@ -97,7 +97,7 @@ def test_given_clone_targets_when_compiling_then_all_projects_use_destination_co local_config=LocalConfig(), ) - result: ClonePipelineResult = prepare_clone_pipeline( + result: ClonePipelineResult = run_clone_pipeline_with_options( discovered_inputs=discovered_inputs, adapter=PostgresAdapter(), origin_target_name="prod", diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py index c58d2ae8dc..bc0e67322a 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py @@ -7,12 +7,13 @@ import pytest -from sqlbuild._native import build_python_plan_entries, read_latest_fingerprints +from sqlbuild._native import ( + build_python_plan_entries, + build_python_sql_run_lifecycle, + read_latest_fingerprints, +) from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.pipeline.models import PythonPlanEntry -from sqlbuild.compiler.python_nodes._helpers.run_lifecycle import ( - build_python_sql_run_lifecycle_plan, -) from sqlbuild.compiler.python_nodes.models import ( PythonNodeGraph, PythonNodeIdentity, @@ -69,7 +70,7 @@ def test_given_previous_python_identity_when_building_plan_entries_then_sets_sta ) entries: tuple[PythonPlanEntry, ...] = build_python_plan_entries( - lifecycle_plan=build_python_sql_run_lifecycle_plan( + lifecycle_plan=build_python_sql_run_lifecycle( selection=PythonSqlRunSelection( sql_keys=frozenset(), python_node_names=frozenset({"prepare_orders"}), @@ -144,7 +145,7 @@ def test_given_filtered_warehouse_fingerprints_when_planning_python_then_sets_id filtered_node_types=("model", "seed", "sql_udf"), ) entries: tuple[PythonPlanEntry, ...] = build_python_plan_entries( - lifecycle_plan=build_python_sql_run_lifecycle_plan( + lifecycle_plan=build_python_sql_run_lifecycle( selection=PythonSqlRunSelection( sql_keys=frozenset(), python_node_names=frozenset({"prepare_orders"}), diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index a3594a3e41..c20ea521fa 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -10,7 +10,6 @@ import sqlbuild._native as _native from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline._helpers import analysis_selection from sqlbuild.compiler.pipeline.main import _plan_result as plan_result from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput @@ -68,9 +67,7 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di monkeypatch.setattr(open_connection, "open_connection_with_hooks", Mock(return_value=object())) monkeypatch.setattr(plan_result, "build_pipeline_result", Mock(return_value=pipeline_result)) monkeypatch.setattr(load_config, "load_rules_config", Mock(return_value=RulesConfig())) - monkeypatch.setattr( - analysis_selection, "resolve_configured_rule_selection", Mock(return_value=None) - ) + monkeypatch.setattr(_native, "resolve_configured_rule_selection", Mock(return_value=None)) monkeypatch.setattr( run_rules, "run_rules", diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index 67848cdf49..c6f621657f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -61,14 +61,6 @@ class DetectFunctionCallerChangeTestCase: expected_backfill: BackfillResult -@dataclass(frozen=True) -class DetectQueryChangeTestCase: - description: str - compiled_query_hash: str - fingerprint_query_hash: str - expected_changed: bool - - @dataclass(frozen=True) class DetectSchemaChangesTestCase: description: str @@ -82,20 +74,6 @@ class DetectSchemaChangesTestCase: dialect: str | None = None -@dataclass(frozen=True) -class ResolveBackfillTestCase: - description: str - raw_value: str | None - expected_result: BackfillResult - - -@dataclass(frozen=True) -class ResolveBackfillErrorTestCase: - description: str - raw_value: str - expected_error_fragment: str - - @dataclass(frozen=True) class DetectRenamedModelTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_declared_columns_metadata.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_declared_columns_metadata.py index ea10347407..9387516453 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_declared_columns_metadata.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_declared_columns_metadata.py @@ -2,10 +2,11 @@ from __future__ import annotations +import json + import pytest -from sqlbuild.compiler.planner._helpers.changes.metadata import ( - recorded_declared_columns_hash, +from sqlbuild._native import ( version_identity_metadata_payload, with_declared_columns_hash, without_declared_columns_hash, @@ -38,7 +39,7 @@ def test_given_fingerprint_metadata_when_reading_declared_columns_hash_then_tolerates_absence( test_case: DeclaredColumnsMetadataTestCase, ) -> None: - assert recorded_declared_columns_hash(test_case.metadata_json) == ( + assert json.loads(test_case.metadata_json).get("declared_columns_hash") == ( test_case.expected_recorded_hash ) assert version_identity_metadata_payload(test_case.metadata_json) == ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 8bdc8b3693..c3c95b77f6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -5,17 +5,18 @@ import pytest -from sqlbuild._native import compute_query_hash -from sqlbuild.compiler.compile.models import InferredColumn -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.changes.detect import detect_changes, detect_model_changes -from sqlbuild.compiler.planner._helpers.output.strategy import ( +from sqlbuild._native import ( build_model_warnings, + compute_query_hash, + detect_changes, + detect_model_changes, get_materialization_type, resolve_model_plan_action, ) +from sqlbuild.compiler.compile.models import InferredColumn +from sqlbuild.compiler.compile.types import CompiledModelView +from sqlbuild.compiler.discovery.models import SqlHookEntry +from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.constants import RECORDED_RELATION_MISSING_WARNING_CODE from sqlbuild.compiler.planner.main.identity._version_identity_metadata import ( build_version_identity_metadata_json, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_policy.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_policy.py deleted file mode 100644 index 2b12491a2d..0000000000 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_policy.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.planner._helpers.changes.policy import resolve_replay_on_change -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import BackfillResult -from sqlbuild.compiler.planner.types import BackfillAction -from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( - ResolveBackfillErrorTestCase, - ResolveBackfillTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - ResolveBackfillTestCase( - description="returns full for full policy", - raw_value="full", - expected_result=BackfillResult(action=BackfillAction.FULL), - ), - ResolveBackfillTestCase( - description="returns bounded with duration for bounded policy", - raw_value="bounded-30d", - expected_result=BackfillResult(action=BackfillAction.BOUNDED, duration="30d"), - ), - ResolveBackfillTestCase( - description="returns forward only for null policy", - raw_value=None, - expected_result=BackfillResult(action=BackfillAction.FORWARD_ONLY), - ), - ], - ids=lambda case: case.description, -) -def test_given_policy_when_resolving_replay_on_change_then_returns_expected( - test_case: ResolveBackfillTestCase, -) -> None: - result: BackfillResult = resolve_replay_on_change( - replay_on_change=test_case.raw_value, - ) - - assert result == test_case.expected_result - - -@pytest.mark.parametrize( - "test_case", - [ - ResolveBackfillErrorTestCase( - description="raises for an unvalidated policy value", - raw_value="unknown", - expected_error_fragment="unknown replay_on_change 'unknown'", - ), - ], - ids=lambda case: case.description, -) -def test_given_unvalidated_policy_when_resolving_replay_on_change_then_raises( - test_case: ResolveBackfillErrorTestCase, -) -> None: - with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment): - resolve_replay_on_change(replay_on_change=test_case.raw_value) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_query.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_query.py deleted file mode 100644 index 6321b4b72a..0000000000 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_query.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import annotations - -from datetime import datetime - -import pytest - -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.changes.query import detect_query_change -from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( - DetectQueryChangeTestCase, -) - -_STUB_TS: datetime = datetime(2026, 1, 15, 12, 0, 0) - - -@pytest.mark.parametrize( - "test_case", - [ - DetectQueryChangeTestCase( - description="detects no change when query hashes match", - compiled_query_hash="abc", - fingerprint_query_hash="abc", - expected_changed=False, - ), - DetectQueryChangeTestCase( - description="detects change when query hashes differ", - compiled_query_hash="abc", - fingerprint_query_hash="def", - expected_changed=True, - ), - ], - ids=lambda case: case.description, -) -def test_given_hashes_when_detecting_query_change_then_returns_expected( - test_case: DetectQueryChangeTestCase, -) -> None: - fingerprint: Fingerprint = Fingerprint( - node_type="model", - node_name="test", - target_database=None, - target_schema=None, - target_name="test", - run_id="run_001", - definition_hash=test_case.fingerprint_query_hash, - schema_fingerprint="schema_a", - definition="SELECT 1", - ts=_STUB_TS, - ) - result: bool = detect_query_change( - compiled_query_hash=test_case.compiled_query_hash, - fingerprint=fingerprint, - ) - - assert result == test_case.expected_changed diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py index da4bc9289d..6e9cae5263 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py @@ -6,9 +6,8 @@ import pytest -from sqlbuild._native import compute_query_hash +from sqlbuild._native import compute_query_hash, origin_reference_names from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.changes.reference_renames import origin_reference_names from sqlbuild.compiler.planner.models import ReferenceRename from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( OriginReferenceNamesTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py index d28471cad1..e99f28e530 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py @@ -2,9 +2,9 @@ import pytest +from sqlbuild._native import detect_schema_changes from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import InferredColumn -from sqlbuild.compiler.planner._helpers.changes.schema import detect_schema_changes from sqlbuild.compiler.planner.models import SchemaFinding from sqlbuild.compiler.planner.types import SchemaChangeKind, SchemaColumnSource from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 61383e156a..4f60c4a0a5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -8,8 +8,8 @@ from datetime import UTC, datetime from itertools import chain from pathlib import Path -from types import MappingProxyType -from typing import ClassVar +from types import MappingProxyType, SimpleNamespace +from typing import Any, ClassVar import sqlbuild._native as _native from sqlbuild._native import ( @@ -83,6 +83,7 @@ from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import ( + DefaultsConfig, SchemaColumn, SchemaModelEntry, SchemaSeedEntry, @@ -1478,9 +1479,19 @@ def native_audit_scope_deps( ], attached, ) - audit_deps: list[list[tuple[str, str]]] = _native.assemble_project_resource_facts( - ("generic", None, (None, None, None, None), [], [], [], [], [], [], [audit_row]) - )[4] + inputs: Any = SimpleNamespace( + effective_vars={}, + effective_target=None, + project_config=SimpleNamespace(defaults=DefaultsConfig()), + seed_inputs=(), + model_inputs=(), + source_inputs=(), + sql_function_inputs=(), + audit_inputs=(SimpleNamespace(assembly_row=audit_row),), + ) + audit_deps: tuple[list[tuple[str, str]], ...] = _native.assemble_native_project_resources( + inputs=inputs, dialect=None, syntax_checks=() + ).audit_scope_deps return tuple( CompiledObjectKey(resource_type=CompiledResourceType(kind), name=name) for kind, name in audit_deps[0] diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py index cbdff45f0a..4264d405d9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py @@ -4,6 +4,11 @@ import pytest +from sqlbuild._native import ( + reads_outside_target_lineage, + resolve_attachment_kind, + resolve_effective_run_scope, +) from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditRunScope, @@ -16,11 +21,6 @@ AttachedAuditTargetKind, CompiledAuditView, ) -from sqlbuild.compiler.planner._helpers.output.audit_scheduling import ( - reads_outside_target_lineage, - resolve_attachment_kind, - resolve_effective_run_scope, -) from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_type_check.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_type_check.py index a13bec6358..8e9db31606 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_type_check.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_type_check.py @@ -4,10 +4,8 @@ import pytest +from sqlbuild._native import check_cursor_type_consistency from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.planner._helpers.output.cursor_type_check import ( - check_cursor_type_consistency, -) from sqlbuild.compiler.planner.models import PlanWarning from sqlbuild.compiler.planner.types import WarningSeverity from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py index 176bf94ee9..02c08d1056 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py @@ -2,8 +2,8 @@ import pytest +from sqlbuild._native import resolve_model_plan_action from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner._helpers.output.strategy import resolve_model_plan_action from sqlbuild.compiler.planner.models import ChangeDetectionResult, SchemaFinding from sqlbuild.compiler.planner.types import ( BackfillAction, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py index 6ee1d5be87..24d986187f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.planner._helpers.output.strategy import resolve_schema_actions +from sqlbuild._native import resolve_schema_actions from sqlbuild.compiler.planner.models import SchemaAction, SchemaFinding from sqlbuild.compiler.planner.types import ( OnSchemaChange, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py index 24d7d390b6..34a4154bea 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.planner._helpers.output.strategy import build_model_warnings +from sqlbuild._native import build_model_warnings from sqlbuild.compiler.planner.models import ( ChangeDetectionResult, PlanWarning, diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py index 016fc0b8cd..6703d65bae 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py @@ -11,8 +11,8 @@ import sqlbuild.compiler.planner.classes.fixture_column_inferences as fixture_inferences import sqlbuild.compiler.planner.classes.migration_fingerprint_cache as fingerprint_cache +from sqlbuild._native import infer_fixture_column_facts from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.main._infer_fixture_columns import infer_fixture_column_facts from sqlbuild.compiler.compile.models import FixtureColumnInference from sqlbuild.compiler.planner._helpers.migrations.fingerprint import build_migration_fingerprint diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_fixture_column_inferences.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_fixture_column_inferences.py index b474b847d8..a731ac9d73 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_fixture_column_inferences.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_fixture_column_inferences.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.compile.main._infer_fixture_columns import infer_fixture_column_facts +from sqlbuild._native import infer_fixture_column_facts from sqlbuild.compiler.compile.models import FixtureColumnInference from sqlbuild.compiler.planner.classes.fixture_column_inferences import FixtureColumnInferences from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py index 93ac2e1b3b..560bcec84c 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py @@ -4,11 +4,9 @@ import pytest +from sqlbuild._native import build_python_sql_run_lifecycle from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.python_nodes._helpers.run_lifecycle import ( - build_python_sql_run_lifecycle_plan, -) from sqlbuild.compiler.python_nodes.models import ( PythonNodeGraph, PythonSqlRunLifecyclePlan, @@ -91,7 +89,7 @@ def test_given_run_selection_when_building_lifecycle_plan_then_classifies_phases ) -> None: graph: PythonNodeGraph = build_python_node_graph_for_case(test_case.python_graph_case) - result: PythonSqlRunLifecyclePlan = build_python_sql_run_lifecycle_plan( + result: PythonSqlRunLifecyclePlan = build_python_sql_run_lifecycle( selection=test_case.selection, python_graph=graph, ) diff --git a/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py index 315de7de6a..96ae930421 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py @@ -8,13 +8,13 @@ from itertools import compress from pathlib import PurePath -from sqlbuild.compiler.scopes._helpers.paths import normalize_path -from sqlbuild.compiler.scopes.main._query_scope_target import query_scope_target -from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( +from sqlbuild._native import ( + build_scope_lookup, + normalize_scope_path, + query_scope_target, resolve_scope_declaration_visibility, + resolve_scope_visibility, ) -from sqlbuild.compiler.scopes.main._resolve_scope_visibility import resolve_scope_visibility -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, @@ -156,7 +156,7 @@ def _targets(lookup: ScopeLookup) -> list[ResourceIdentity | str]: def _parent(path: str) -> str: - return posixpath.dirname(normalize_path(path=path)) or "." + return posixpath.dirname(normalize_scope_path(path=path)) or "." def _is_descendant(*, path: str, ancestor: str) -> bool: @@ -166,7 +166,7 @@ def _is_descendant(*, path: str, ancestor: str) -> bool: def _reference_reason( *, resource: ResourceRecord, declaration: DeclarationRecord ) -> VisibilityReason | None: - owner: str = normalize_path(path=declaration.owning_path or ".") + owner: str = normalize_scope_path(path=declaration.owning_path or ".") parent: str = _parent(resource.path) visible: dict[ScopeKind, bool] = { ScopeKind.GLOBAL: True, @@ -180,7 +180,7 @@ def _reference_reason( def _reference_inaccessible_reason( *, resource: ResourceRecord, declaration: DeclarationRecord ) -> InaccessibleReason: - owner: str = normalize_path(path=declaration.owning_path or ".") + owner: str = normalize_scope_path(path=declaration.owning_path or ".") candidates: tuple[tuple[bool, InaccessibleReason], ...] = ( (declaration.scope is ScopeKind.PRIVATE, InaccessibleReason.PRIVATE_OWNER), (declaration.scope is ScopeKind.LOCAL, InaccessibleReason.LOCAL_BOUNDARY), diff --git a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py index e420ec69c1..36c6d58db8 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py @@ -7,15 +7,15 @@ from pathlib import Path from typing import cast +from sqlbuild._native import ( + build_scope_lookup, + get_placement_validated_scope_index, +) from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, DiscoveredSqlModelFile, EnumDeclaration, ) -from sqlbuild.compiler.scopes.main._get_placement_validated_scope_index import ( - get_placement_validated_scope_index, -) -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup from sqlbuild.compiler.scopes.models import ( ConstantMetadata, DeclarationIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py index 2e7c130848..435c399591 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py @@ -6,6 +6,14 @@ import pytest +from sqlbuild._native import ( + build_scope_index, + build_scope_lookup, + normalize_sql_value, + query_scope_target, + resolve_scope_visibility, + validate_scope_index, +) from sqlbuild.compiler.compile.models import LoadedMacro from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, @@ -24,11 +32,6 @@ EnumMember, ) from sqlbuild.compiler.scopes.exceptions import ScopeValidationError -from sqlbuild.compiler.scopes.main._build_scope_index import build_scope_index -from sqlbuild.compiler.scopes.main._query_scope_target import query_scope_target -from sqlbuild.compiler.scopes.main._resolve_scope_visibility import resolve_scope_visibility -from sqlbuild.compiler.scopes.main._validate_scope_index import validate_scope_index -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, @@ -49,7 +52,6 @@ VisibilityReason, ) from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.sql_values.main.normalize import normalize_sql_value from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( ExpectedBooleanCase, ResourceExpectation, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py b/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py index e8263cb7de..49ea1b4695 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py @@ -4,10 +4,8 @@ import pytest +from sqlbuild._native import build_scope_lookup, explain_scope_declaration, query_scope_report from sqlbuild.compiler.scopes.exceptions import InvalidQualifiedIdentityError -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup -from sqlbuild.compiler.scopes.main.explain_scope_declaration import explain_scope_declaration -from sqlbuild.compiler.scopes.main.query_scope_report import query_scope_report from sqlbuild.compiler.scopes.models import ( DeclarationExplanation, DeclarationIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py b/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py index 762a66aa14..700a8bf189 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py @@ -8,8 +8,7 @@ import pytest -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup -from sqlbuild.compiler.scopes.main.scope_metadata import scope_metadata_projection +from sqlbuild._native import build_scope_lookup, scope_metadata_projection from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py b/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py index 7abb2d4126..2542649660 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.scopes._helpers.paths import normalize_path +from sqlbuild._native import normalize_scope_path from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError from tests.unit.src.sqlbuild.compiler.scopes._test_types import PathNormalizationCase @@ -33,7 +33,7 @@ def test_given_posix_or_windows_path_when_normalizing_then_returns_canonical_posix_path( test_case: PathNormalizationCase, ) -> None: - assert normalize_path(path=test_case.path) == test_case.expected_path + assert normalize_scope_path(path=test_case.path) == test_case.expected_path @pytest.mark.parametrize( @@ -49,7 +49,7 @@ def test_given_path_outside_project_when_normalizing_then_rejects_it( test_case: PathNormalizationCase, ) -> None: with pytest.raises(InvalidScopePathError, match=test_case.expected_path): - normalize_path(path=test_case.path) + normalize_scope_path(path=test_case.path) if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py b/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py index 03fdcdb665..532f665da8 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py @@ -4,11 +4,11 @@ import pytest -from sqlbuild.compiler.scopes.exceptions import ScopeValidationError -from sqlbuild.compiler.scopes.main._get_placement_validated_scope_index import ( +from sqlbuild._native import ( get_placement_validated_scope_index, + validate_scope_index, ) -from sqlbuild.compiler.scopes.main._validate_scope_index import validate_scope_index +from sqlbuild.compiler.scopes.exceptions import ScopeValidationError from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py b/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py index 8f76f0ce39..0a508e3933 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.scopes.main.relocate_declarations_for_move import ( +from sqlbuild._native import ( relocate_declarations_for_move, ) from sqlbuild.compiler.scopes.models import ( diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_reports.py b/tests/unit/src/sqlbuild/compiler/scopes/test_reports.py index f540473037..79ea620d89 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_reports.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_reports.py @@ -9,13 +9,15 @@ import pytest -from sqlbuild.compiler.scopes.main.browse_scope_folders import browse_scope_folders -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup -from sqlbuild.compiler.scopes.main.explain_scope_declaration import explain_scope_declaration -from sqlbuild.compiler.scopes.main.list_scope_declarations import list_scope_declarations -from sqlbuild.compiler.scopes.main.preview_scope_move import preview_scope_move -from sqlbuild.compiler.scopes.main.query_scope_report import query_scope_report -from sqlbuild.compiler.scopes.main.serialize_scope_report import serialize_scope_report +from sqlbuild._native import ( + browse_scope_folders, + build_scope_lookup, + explain_scope_declaration, + list_scope_declarations, + preview_scope_move, + query_scope_report, + serialize_scope_report, +) from sqlbuild.compiler.scopes.models import ( OwnershipRoot, ResourceRecord, diff --git a/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py b/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py index 4f1ef80581..a2dd5fd97d 100644 --- a/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py +++ b/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py @@ -6,8 +6,11 @@ from scripts.setting_help.main.find_settings_without_snippet import find_settings_without_snippet from scripts.setting_help.main.parse_setting_snippet import parse_setting_snippet -from sqlbuild._native import explicit_references_help, microbatch_guidance -from sqlbuild.compiler.compile._helpers.analysis.validation import sql_analysis_opt_out_help +from sqlbuild._native import ( + explicit_references_help, + microbatch_guidance, + sql_analysis_opt_out_help, +) from sqlbuild.errors.setting_help.main.join_helps import join_helps from sqlbuild.errors.setting_help.main.model_header_help import model_header_help from sqlbuild.errors.setting_help.main.setting_help import setting_help diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py index 02f9f3bdb6..e3cb4245ec 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py @@ -18,9 +18,9 @@ import pytest +from sqlbuild._native import AuditProgressReporter from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.progress.classes.audit_progress_reporter import AuditProgressReporter from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditOutcome, diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index 5b38105589..5647bb96ac 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import Any, ClassVar, cast +from sqlbuild._native import build_python_sql_run_lifecycle from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.assets import AssetContext, asset @@ -22,9 +23,6 @@ DiscoveredProjectInputs, DiscoveredTaskFunction, ) -from sqlbuild.compiler.python_nodes._helpers.run_lifecycle import ( - build_python_sql_run_lifecycle_plan, -) from sqlbuild.compiler.python_nodes._helpers.user_inventory import build_python_node_graph from sqlbuild.compiler.python_nodes.models import ( DiscoveredPythonNode, @@ -423,7 +421,7 @@ def assert_base_context_fields(context: BasePythonNodeContext) -> None: def build_lifecycle_plan_for_selected_python_names( *, graph: PythonNodeGraph, selected_names: frozenset[str] ) -> PythonSqlRunLifecyclePlan: - return build_python_sql_run_lifecycle_plan( + return build_python_sql_run_lifecycle( selection=PythonSqlRunSelection(sql_keys=frozenset(), python_node_names=selected_names), python_graph=graph, ) diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_event_stream.py b/tests/unit/src/sqlbuild/integrations/dbt/test_event_stream.py index dc27cf2af1..eec7cdfffe 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_event_stream.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_event_stream.py @@ -888,21 +888,6 @@ def test_given_active_dbt_node_when_dbt_stream_is_silent_then_status_elapsed_upd test_case: DbtSilentStatusRefreshTestCase, monkeypatch: pytest.MonkeyPatch, ) -> None: - status_messages: list[str] = [] - - class CapturingStatusReporter: - def __init__(self, *args: object, **kwargs: object) -> None: - pass - - def start(self, message: str) -> None: - status_messages.append(message) - - def update(self, message: str) -> None: - status_messages.append(message) - - def close(self) -> None: - pass - class TtyStream(io.StringIO): def isatty(self) -> bool: return True @@ -934,10 +919,7 @@ class StubProcess: def wait(self) -> int: return 0 - monkeypatch.setattr( - "sqlbuild.integrations.dbt._helpers.runtime.event_stream.TransientStatusReporter", - CapturingStatusReporter, - ) + monkeypatch.setenv("SQLBUILD_NO_PROGRESS", "1") monkeypatch.setattr( "sqlbuild.integrations.dbt._helpers.runtime.event_stream._DBT_STATUS_REFRESH_SECONDS", test_case.refresh_seconds, @@ -947,18 +929,20 @@ def wait(self) -> int: lambda *args, **kwargs: StubProcess(), ) + stream: TtyStream = TtyStream() returncode: int returncode, _ = execute_dbt_json_event_stream( argv=("dbt", "run"), cwd=None, - stream=TtyStream(), + stream=stream, use_color=False, target_path=None, ) + status_lines: list[str] = stream.getvalue().splitlines() assert returncode == 0 - assert test_case.expected_initial_status in status_messages - assert test_case.expected_refreshed_status in status_messages + assert test_case.expected_initial_status in status_lines + assert test_case.expected_refreshed_status in status_lines @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py index 5577218e17..8dafbf4b74 100644 --- a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py +++ b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py @@ -8,7 +8,7 @@ import pytest -import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module +import sqlbuild._native as native_module import sqlbuild.compiler.planner._helpers.sql_tests.fixture_formatting as fixture_formatting_module from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -64,10 +64,10 @@ def test_given_value_renderer_when_building_expansion_context_then_scope_uses_it monkeypatch: pytest.MonkeyPatch, ) -> None: _ = write_fixture_format_project(tmp_path=tmp_path, fixture_sql="SELECT 1 AS id, TRUE AS flag") - build_scope: Mock = Mock(wraps=expansion_context_module.build_declaration_scope) - monkeypatch.setattr(expansion_context_module, "build_declaration_scope", build_scope) + build_scope: Mock = Mock(wraps=native_module.build_declaration_scope) + monkeypatch.setattr(native_module, "build_declaration_scope", build_scope) - _ = expansion_context_module.build_sql_expansion_context( + _ = native_module.build_sql_expansion_context( project_dir=tmp_path, value_renderer=_CustomSyntaxDuckDbAdapter() ) diff --git a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py index 411ca2e86f..6b68156b5c 100644 --- a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py @@ -7,13 +7,16 @@ import pytest -from sqlbuild._native import run_lint -from sqlbuild.compiler.compile.main.expand_sql_with_spans import expand_sql_with_spans -from sqlbuild.compiler.compile.main.map_expanded_offset import map_expanded_offset +import sqlbuild._native as native_module +from sqlbuild._native import ( + build_lint_expansion_context, + expand_sql_with_spans, + map_expanded_offset, + run_lint, +) from sqlbuild.compiler.compile.models import MappedOffset, SqlExpansionContext from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.lint._helpers import expansion -from sqlbuild.lint._helpers.expansion import build_lint_expansion_context, prepare_lint_body +from sqlbuild.lint._helpers.expansion import prepare_lint_body from sqlbuild.lint._helpers.headers import lint_file_role from sqlbuild.lint.exceptions import ProjectCompileError from sqlbuild.lint.models import LintBody, LintConfig, LintRunResult @@ -42,7 +45,7 @@ def test_given_lint_context_when_building_then_project_is_discovered_once( ) -> None: _ = (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") discovery_count: list[int] = [0] - discover: Callable[..., DiscoveredProjectInputs] = expansion.discover_project_inputs + discover: Callable[..., DiscoveredProjectInputs] = native_module.discover_project_inputs def counted_discovery( *, @@ -57,7 +60,7 @@ def counted_discovery( extract_output_column_locations=extract_output_column_locations, ) - monkeypatch.setattr(expansion, "discover_project_inputs", counted_discovery) + monkeypatch.setattr(native_module, "discover_project_inputs", counted_discovery) _ = build_lint_expansion_context(project_dir=tmp_path) diff --git a/tests/unit/src/sqlbuild/presentation/classes/_test_types.py b/tests/unit/src/sqlbuild/presentation/classes/_test_types.py index bedaf3ccc2..89c32a2216 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/presentation/classes/_test_types.py @@ -45,3 +45,17 @@ class TransientLineCoordinatorTestCase: description: str persistent_stream_is_tty: bool expected_events: tuple[str, ...] + + +@dataclass(frozen=True) +class StatusReporterOutputTestCase: + description: str + use_color: bool + tty: bool + expected_persistent_lines: tuple[str, ...] + + +@dataclass(frozen=True) +class StatusReporterExitTestCase: + description: str + expected_returncode: int diff --git a/tests/unit/src/sqlbuild/presentation/classes/helpers.py b/tests/unit/src/sqlbuild/presentation/classes/helpers.py index 8551ac1c9e..ebb758f669 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/helpers.py +++ b/tests/unit/src/sqlbuild/presentation/classes/helpers.py @@ -1,12 +1,16 @@ from __future__ import annotations import io +import os +import re from collections.abc import Callable from types import MappingProxyType from typing import cast import pytest +from sqlbuild._native import TransientStatusReporter + def _delete_no_progress_env(monkeypatch: pytest.MonkeyPatch, env_value: str | None) -> None: del env_value @@ -53,3 +57,44 @@ def clear_transient_line(self) -> None: def redraw_transient_line(self) -> None: self._events.append("redraw") + + +def drive_status_reporter(reporter: TransientStatusReporter) -> None: + """Run one start, update, completion, preflight and error sequence.""" + + reporter.start("Discovering project...") + reporter.update("Compiling project graph...") + reporter.complete(message="Compiled project graph. (0.1s)", blank_line_after=True) + reporter.report_preflight_progress("Resolving tests...") + reporter.report_preflight_progress("Prepared 2 tests") + reporter.start("Formatting SQL...") + reporter.error("Formatting SQL ERROR") + reporter.complete_styled(message="styled") + reporter.close() + + +def persistent_lines(text: str) -> tuple[str, ...]: + """The lines left after transient status lines are erased.""" + + without_cursor: str = re.sub(r"\x1b\[\?25[hl]", "", text) + return tuple(line.rsplit("\x1b[K", 1)[-1] for line in without_cursor.split("\n")) + + +OPEN_SPINNER_SCRIPT: str = ( + "import io, time\n" + "from sqlbuild._native import TransientStatusReporter\n" + "class Tty(io.StringIO):\n" + " def isatty(self):\n" + " return True\n" + "reporter = TransientStatusReporter(stream=Tty())\n" + "reporter.start('Working...')\n" + "time.sleep(0.35)\n" +) + + +def without_no_progress_env() -> dict[str, str]: + """The current environment with progress spinners left enabled.""" + + environment: dict[str, str] = dict(os.environ) + environment.pop("SQLBUILD_NO_PROGRESS", None) + return environment diff --git a/tests/unit/src/sqlbuild/presentation/classes/test_status.py b/tests/unit/src/sqlbuild/presentation/classes/test_status.py index 2defadc1c5..9f0c1e5db5 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/test_status.py +++ b/tests/unit/src/sqlbuild/presentation/classes/test_status.py @@ -1,15 +1,26 @@ from __future__ import annotations import io +import subprocess +import sys import pytest -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter +from sqlbuild._native import TransientStatusReporter from sqlbuild.presentation.main._progress_spinners_disabled import progress_spinners_disabled from tests.unit.src.sqlbuild.presentation.classes._test_types import ( ProgressSpinnersDisabledTestCase, + StatusReporterExitTestCase, + StatusReporterOutputTestCase, +) +from tests.unit.src.sqlbuild.presentation.classes.helpers import ( + OPEN_SPINNER_SCRIPT, + RecordingStream, + apply_no_progress_env, + drive_status_reporter, + persistent_lines, + without_no_progress_env, ) -from tests.unit.src.sqlbuild.presentation.classes.helpers import apply_no_progress_env class _TtyStream(io.StringIO): @@ -116,6 +127,85 @@ def test_given_no_progress_env_when_building_status_on_tty_then_toggles_spinner( stream: _TtyStream = _TtyStream() reporter: TransientStatusReporter = TransientStatusReporter(stream=stream, use_color=False) + reporter.start("Working...") + reporter.close() + + assert ("Working...\n" in stream.getvalue()) == test_case.expected_disabled + + +@pytest.mark.parametrize( + "test_case", + ( + StatusReporterOutputTestCase( + description="plain stream writes every message as a muted line", + use_color=True, + tty=False, + expected_persistent_lines=( + "\x1b[2mDiscovering project...\x1b[0m", + "\x1b[2mCompiling project graph...\x1b[0m", + "\x1b[2mCompiled project graph. (0.1s)\x1b[0m", + "", + "\x1b[2mResolving tests...\x1b[0m", + "\x1b[2mPrepared 2 tests\x1b[0m", + "", + "\x1b[2mFormatting SQL...\x1b[0m", + "Formatting SQL ERROR", + "styled", + "", + ), + ), + StatusReporterOutputTestCase( + description="terminal erases live status and keeps only completions", + use_color=False, + tty=True, + expected_persistent_lines=( + "Compiled project graph. (0.1s)", + "", + "Prepared 2 tests", + "", + "Formatting SQL ERROR", + "styled", + "", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_status_sequence_when_reporting_then_writes_expected_persistent_lines( + test_case: StatusReporterOutputTestCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("SQLBUILD_NO_PROGRESS", raising=False) + stream: RecordingStream = RecordingStream(events=[], tty=test_case.tty) + reporter: TransientStatusReporter = TransientStatusReporter( + stream=stream, use_color=test_case.use_color + ) + + drive_status_reporter(reporter) + + assert persistent_lines(stream.getvalue()) == test_case.expected_persistent_lines + + +@pytest.mark.parametrize( + "test_case", + ( + StatusReporterExitTestCase( + description="an open spinner is stopped before the interpreter exits", + expected_returncode=0, + ), + ), + ids=lambda case: case.description, +) +def test_given_open_spinner_when_process_exits_then_exits_cleanly( + test_case: StatusReporterExitTestCase, +) -> None: + completed: subprocess.CompletedProcess[str] = subprocess.run( + (sys.executable, "-c", OPEN_SPINNER_SCRIPT), + capture_output=True, + text=True, + check=False, + env=without_no_progress_env(), + timeout=60, + ) - spinner_enabled: bool = reporter._enabled - assert spinner_enabled == (not test_case.expected_disabled) + assert completed.returncode == test_case.expected_returncode, completed.stderr diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_scope_facts.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_scope_facts.py index 8c7b878fdf..c35037745b 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_scope_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_scope_facts.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild.compiler.scopes.main.scope_metadata import scope_metadata_projection +from sqlbuild._native import scope_metadata_projection from sqlbuild.compiler.scopes.models import DeclarationRecord, ScopeCompleteness, ScopeIndex from tests.unit.src.sqlbuild.compiler.scopes.helpers import scope_index from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ScopePayloadTestCase diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py index 81b9d91e6d..8719db8a69 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py @@ -5,9 +5,9 @@ from sqlbuild._native import ( declaration_rows, + normalize_sql_value, ) from sqlbuild.compiler.discovery.models import ConstantDeclaration -from sqlbuild.sql_values.main.normalize import normalize_sql_value from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( TypedConstantPayloadTestCase, ) diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py index 79b1ccf7cd..77842a9979 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py @@ -2,9 +2,7 @@ import pytest -from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( - resolve_effective_collection_rendering, -) +from sqlbuild._native import resolve_effective_collection_rendering from sqlbuild.spec.contracts.models import ConstantsConfig, ProjectConfig from sqlbuild.sql_values.types import CollectionRendering from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py index f324441d7c..359873a767 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import loader_destination_parts from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.main.loader_destination_parts import loader_destination_parts from sqlbuild.spec.contracts.models import LoaderDestinationParts from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( InvalidLoaderDestinationTestCase, diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py index d54a6514fe..ce278ee9a6 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py @@ -6,12 +6,9 @@ import pytest -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import discover_project_inputs, resolve_effective_scenario_config from sqlbuild.compiler.discovery.exceptions import ProjectConfigError from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.main.resolve_effective_scenario_config import ( - resolve_effective_scenario_config, -) from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( InvalidScenarioNamespaceConfigTestCase, ScenarioNamespaceConfigTestCase, diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py index c8a5971a79..f3ce3f24f0 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py @@ -2,8 +2,8 @@ import pytest +from sqlbuild._native import resolve_target_config from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.models import ( AuthoredTimeTravelRetention, ClonePolicy, diff --git a/tests/unit/src/sqlbuild/sql_values/_test_types.py b/tests/unit/src/sqlbuild/sql_values/_test_types.py index 31b6b8391c..f022acce9c 100644 --- a/tests/unit/src/sqlbuild/sql_values/_test_types.py +++ b/tests/unit/src/sqlbuild/sql_values/_test_types.py @@ -34,14 +34,6 @@ class SqlValueBehaviorTestCase: expected_identities_equal: bool = False -@dataclass(frozen=True) -class RenderedSqlValueLimitTestCase: - description: str - rendered_sql: str - max_size: int - expected_size: int - - @dataclass(frozen=True) class StateLiteralGoldenTestCase: description: str diff --git a/tests/unit/src/sqlbuild/sql_values/test_normalization.py b/tests/unit/src/sqlbuild/sql_values/test_normalization.py index f2aaf11cec..d42e56aa9b 100644 --- a/tests/unit/src/sqlbuild/sql_values/test_normalization.py +++ b/tests/unit/src/sqlbuild/sql_values/test_normalization.py @@ -5,14 +5,12 @@ import pytest +from sqlbuild._native import normalize_sql_value from sqlbuild.sql_values.exceptions import SqlValueValidationError -from sqlbuild.sql_values.main.normalize import normalize_sql_value -from sqlbuild.sql_values.main.validate_rendered_size import validate_rendered_sql_value_size from sqlbuild.sql_values.models import AuthoredSqlSet, SqlValue, SqlValueLimits from tests.unit.src.sqlbuild.sql_values._test_types import ( NormalizeSqlValueErrorTestCase, NormalizeSqlValueTestCase, - RenderedSqlValueLimitTestCase, SqlValueBehaviorTestCase, SqlValueLimitTestCase, ) @@ -150,31 +148,5 @@ def test_given_safety_limit_is_exceeded_when_normalizing_then_rejects_value( ) -@pytest.mark.parametrize( - "test_case", - [ - RenderedSqlValueLimitTestCase( - description="multibyte rendered SQL uses UTF-8 byte length", - rendered_sql="é", - max_size=1, - expected_size=2, - ) - ], - ids=lambda case: case.description, -) -def test_given_rendered_sql_exceeds_limit_when_validating_then_counts_utf8_bytes( - test_case: RenderedSqlValueLimitTestCase, -) -> None: - with pytest.raises( - SqlValueValidationError, - match=rf"rendered value is {test_case.expected_size} bytes", - ): - validate_rendered_sql_value_size( - rendered_sql=test_case.rendered_sql, - context="constant 'limited'", - max_size=test_case.max_size, - ) - - if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/sql_values/test_state_literal.py b/tests/unit/src/sqlbuild/sql_values/test_state_literal.py index a9249985fc..7f6f3a5932 100644 --- a/tests/unit/src/sqlbuild/sql_values/test_state_literal.py +++ b/tests/unit/src/sqlbuild/sql_values/test_state_literal.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal +from sqlbuild._native import render_state_sql_literal from sqlbuild.sql_values.types import StateSqlValueType from tests.unit.src.sqlbuild.sql_values._test_types import StateLiteralGoldenTestCase