From 09d174699dc7b0a5936cf1194c7509f853536e05 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 03:14:05 +0100 Subject: [PATCH 01/40] refactor: render build progress rows and footer in sqlbuild-core --- .../src/argv/_helpers/formatting.rs | 10 +- .../src/argv/tests/test_text_wrap.rs | 3 +- .../src/build_progress/_helpers/audits.rs | 92 +++++ .../src/build_progress/_helpers/error_rows.rs | 83 +++++ .../src/build_progress/_helpers/footer.rs | 344 ++++++++++++++++++ .../src/build_progress/_helpers/mod.rs | 6 + .../src/build_progress/_helpers/model_rows.rs | 248 +++++++++++++ .../src/build_progress/_helpers/rows.rs | 82 +++++ .../src/build_progress/_helpers/text.rs | 142 ++++++++ .../src/build_progress/constants.rs | 37 ++ .../main/format_build_footer.rs | 57 +++ .../src/build_progress/main/mod.rs | 9 + .../main/progress_name_width.rs | 16 + .../main/query_diagnostic_message.rs | 20 + .../main/render_build_spinner_line.rs | 32 ++ .../main/render_failed_test_result.rs | 54 +++ .../main/render_model_result.rs | 44 +++ .../main/render_resource_result.rs | 82 +++++ .../main/runtime_diagnostic_summary.rs | 28 ++ .../main/scheduler_diagnostic_message.rs | 24 ++ .../sqlbuild-core/src/build_progress/mod.rs | 9 + .../src/build_progress/models.rs | 211 +++++++++++ .../sqlbuild-core/src/build_progress/tests.rs | 6 + .../src/build_progress/tests/helpers.rs | 57 +++ .../tests/test_build_progress.rs | 295 +++++++++++++++ .../src/build_progress/tests/test_types.rs | 52 +++ .../src/cli_text/_helpers/coded_error.rs | 66 +++- .../sqlbuild-core/src/cli_text/constants.rs | 2 + .../main/format_inline_error_lines.rs | 21 ++ crates/sqlbuild-core/src/cli_text/main/mod.rs | 1 + .../format_command/_helpers/diagnostics.rs | 1 + crates/sqlbuild-core/src/lib.rs | 1 + .../src/text/_helpers/text_wrap.rs | 43 ++- .../sqlbuild-core/src/text/main/text_wrap.rs | 13 +- .../src/text/tests/test_text_wrap.rs | 2 +- .../_helpers/build_progress/footer.rs | 36 ++ .../bindings/_helpers/build_progress/mod.rs | 3 + .../_helpers/build_progress/registration.rs | 9 + .../_helpers/build_progress/result_text.rs | 243 +++++++++++++ .../bindings/_helpers/cli/progress_text.rs | 2 +- .../src/bindings/_helpers/cli/registration.rs | 9 +- .../src/bindings/_helpers/mod.rs | 1 + .../bindings/_helpers/plan_output/display.rs | 2 +- .../_helpers/sql_test_command/test_command.rs | 2 +- src/sqlbuild/_native.pyi | 7 + 45 files changed, 2486 insertions(+), 21 deletions(-) create mode 100644 crates/sqlbuild-core/src/build_progress/_helpers/audits.rs create mode 100644 crates/sqlbuild-core/src/build_progress/_helpers/error_rows.rs create mode 100644 crates/sqlbuild-core/src/build_progress/_helpers/footer.rs create mode 100644 crates/sqlbuild-core/src/build_progress/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/build_progress/_helpers/model_rows.rs create mode 100644 crates/sqlbuild-core/src/build_progress/_helpers/rows.rs create mode 100644 crates/sqlbuild-core/src/build_progress/_helpers/text.rs create mode 100644 crates/sqlbuild-core/src/build_progress/constants.rs create mode 100644 crates/sqlbuild-core/src/build_progress/main/format_build_footer.rs create mode 100644 crates/sqlbuild-core/src/build_progress/main/mod.rs create mode 100644 crates/sqlbuild-core/src/build_progress/main/progress_name_width.rs create mode 100644 crates/sqlbuild-core/src/build_progress/main/query_diagnostic_message.rs create mode 100644 crates/sqlbuild-core/src/build_progress/main/render_build_spinner_line.rs create mode 100644 crates/sqlbuild-core/src/build_progress/main/render_failed_test_result.rs create mode 100644 crates/sqlbuild-core/src/build_progress/main/render_model_result.rs create mode 100644 crates/sqlbuild-core/src/build_progress/main/render_resource_result.rs create mode 100644 crates/sqlbuild-core/src/build_progress/main/runtime_diagnostic_summary.rs create mode 100644 crates/sqlbuild-core/src/build_progress/main/scheduler_diagnostic_message.rs create mode 100644 crates/sqlbuild-core/src/build_progress/mod.rs create mode 100644 crates/sqlbuild-core/src/build_progress/models.rs create mode 100644 crates/sqlbuild-core/src/build_progress/tests.rs create mode 100644 crates/sqlbuild-core/src/build_progress/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/build_progress/tests/test_build_progress.rs create mode 100644 crates/sqlbuild-core/src/build_progress/tests/test_types.rs create mode 100644 crates/sqlbuild-core/src/cli_text/main/format_inline_error_lines.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs diff --git a/crates/sqlbuild-cli/src/argv/_helpers/formatting.rs b/crates/sqlbuild-cli/src/argv/_helpers/formatting.rs index a8a487eb9a..fc5e1b4606 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/formatting.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/formatting.rs @@ -136,6 +136,7 @@ fn format_text(formatter: &Formatter, text: &str, indent: i64) -> String { &normalize_whitespace(text), width as usize, (&padding, &padding), + true, ); format!("{}\n\n", lines.join("\n")) } @@ -185,7 +186,14 @@ fn format_action(formatter: &Formatter, entry: &HelpEntry, indent: i64) -> Strin } }; let lines: Vec = help - .map(|help| text_wrap(&normalize_whitespace(help), help_width as usize, ("", ""))) + .map(|help| { + text_wrap( + &normalize_whitespace(help), + help_width as usize, + ("", ""), + true, + ) + }) .unwrap_or_default(); if let Some((first, rest)) = lines.split_first() { text.push_str(&format!("{}{first}\n", " ".repeat(indent_first as usize))); diff --git a/crates/sqlbuild-cli/src/argv/tests/test_text_wrap.rs b/crates/sqlbuild-cli/src/argv/tests/test_text_wrap.rs index ce6cb6fe10..53eac1b414 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_text_wrap.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_text_wrap.rs @@ -34,7 +34,8 @@ fn given_help_text_when_wrapping_then_lines_match_python_textwrap() { text_wrap( &normalize_whitespace(test_case.text), test_case.width, - (test_case.indent, test_case.indent) + (test_case.indent, test_case.indent), + true ), test_case.expected_lines, "{}", diff --git a/crates/sqlbuild-core/src/build_progress/_helpers/audits.rs b/crates/sqlbuild-core/src/build_progress/_helpers/audits.rs new file mode 100644 index 0000000000..aeb8401a10 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/_helpers/audits.rs @@ -0,0 +1,92 @@ +//! Per-batch delta audit results folded into one display row per audit and phase. + +use crate::build_progress::constants::{DELTA_AND_FINAL, ERROR, INSUFFICIENT, PASS, WARN}; +use crate::build_progress::models::AuditResultText; + +/// One aggregated audit row. +pub(crate) struct AuditDisplayEntry { + pub(crate) label: &'static str, + pub(crate) display_name: String, + pub(crate) outcome: &'static str, + pub(crate) total_row_count: i64, + pub(crate) batch_pass: usize, + pub(crate) batch_total: usize, + pub(crate) reused: bool, + pub(crate) executed_sql: Option, +} + +/// Audits sharing a name, column and run-scope phase. +type AuditGroup<'a> = ( + (&'a str, Option<&'a str>, &'a str), + Vec<&'a AuditResultText>, +); + +fn worst_outcome(results: &[&AuditResultText]) -> &'static str { + for outcome in [ERROR, WARN, INSUFFICIENT] { + if results.iter().any(|result| result.outcome == outcome) { + return outcome; + } + } + PASS +} + +fn phase_label(has_delta: bool, phase: &str) -> &'static str { + match (has_delta, phase == DELTA_AND_FINAL) { + (false, _) => "audit", + (true, true) => "audit (d)", + (true, false) => "audit (f)", + } +} + +fn display_entry( + (name, column, phase): (&str, Option<&str>, &str), + results: &[&AuditResultText], + has_delta: bool, +) -> AuditDisplayEntry { + let mut total_row_count: i64 = 0; + let mut batch_pass: usize = 0; + let mut reused: bool = true; + for result in results { + total_row_count += result.row_count; + batch_pass += usize::from(result.outcome == PASS); + reused = reused && result.reused; + } + AuditDisplayEntry { + label: phase_label(has_delta, phase), + display_name: match column { + Some(column) => format!("{name} ({column})"), + None => name.to_owned(), + }, + outcome: worst_outcome(results), + total_row_count, + batch_pass, + batch_total: results.len(), + reused, + executed_sql: results + .first() + .and_then(|result| result.executed_sql.clone()), + } +} + +/// Group audits by name, column and run-scope phase in first-seen order. +pub(crate) fn aggregate_audit_results(audits: &[AuditResultText]) -> Vec { + let has_delta: bool = audits + .iter() + .any(|audit| audit.run_scope_phase == DELTA_AND_FINAL); + let mut groups: Vec> = Vec::new(); + for audit in audits { + let key = ( + audit.audit_name.as_str(), + audit.attached_column_name.as_deref(), + audit.run_scope_phase.as_str(), + ); + match groups.iter_mut().find(|(existing, _)| *existing == key) { + Some((_, members)) => members.push(audit), + None => groups.push((key, vec![audit])), + } + } + groups + .into_iter() + .map(|(key, results)| display_entry(key, &results, has_delta)) + .collect() +} diff --git a/crates/sqlbuild-core/src/build_progress/_helpers/error_rows.rs b/crates/sqlbuild-core/src/build_progress/_helpers/error_rows.rs new file mode 100644 index 0000000000..0f354fa692 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/_helpers/error_rows.rs @@ -0,0 +1,83 @@ +//! Inline error rows under progress lines and error blocks of the failure summary. + +use crate::build_progress::constants::TYPE_WIDTH; +use crate::build_progress::models::ErrorText; +use crate::cli_text::main::format_coded_error::format_coded_error; +use crate::cli_text::main::format_inline_error_lines::format_inline_error_lines; +use crate::cli_text::main::format_tree_connector::format_tree_connector; +use crate::cli_text::models::CliStyle; +use crate::text::main::python_splitlines::python_splitlines; + +/// The `error` label and wrapped message under a row, optionally drawn as a tree child. +pub(crate) fn error_detail( + (error, message): (&ErrorText, &str), + (prefix_width, columns): (usize, usize), + tree_child: bool, + style: CliStyle, +) -> String { + let pad: String = " ".repeat(prefix_width + if tree_child { 4 } else { 0 }); + let connector_suffix: String = if tree_child { + format!("{} ", format_tree_connector(style, true)) + } else { + String::new() + }; + let label_width: usize = if tree_child { + TYPE_WIDTH - 3 + } else { + TYPE_WIDTH - 1 + }; + let label: String = format!( + "{}{}", + style.error_muted("error"), + " ".repeat(label_width.saturating_sub("error".len())) + ); + let plain_prefix_width: usize = pad.len() + if tree_child { 4 } else { 0 } + label_width + 1; + let content_width: usize = (columns as i64 - plain_prefix_width as i64).max(1) as usize; + let lines: Vec = format_inline_error_lines( + (error.code.as_deref(), message, error.help.as_deref()), + content_width, + style, + ); + let continuation: String = " ".repeat(plain_prefix_width); + let mut text: String = String::new(); + for (index, line) in lines.iter().enumerate() { + if index == 0 { + text.push_str(&format!("{pad}{connector_suffix}{label} {line}\n")); + } else { + text.push_str(&format!("{continuation}{line}\n")); + } + } + text +} + +/// The labelled, indented error block of one failure summary entry. +pub(crate) fn failure_error_block( + (code, message, help): (Option<&str>, &str, Option<&str>), + style: CliStyle, +) -> Vec { + let label: String = format!( + "{}{}", + style.error_muted("error"), + " ".repeat(TYPE_WIDTH - "error".len()) + ); + let rendered: String = match code { + None => message.to_owned(), + Some(code) => format_coded_error((code, message, help), false, style), + }; + let mut lines: Vec<&str> = python_splitlines(&rendered); + if lines.is_empty() { + lines.push(&rendered); + } + lines + .iter() + .enumerate() + .map(|(index, line)| { + let display_label: String = if index == 0 { + label.clone() + } else { + " ".repeat(TYPE_WIDTH) + }; + format!(" {display_label}{line}") + }) + .collect() +} diff --git a/crates/sqlbuild-core/src/build_progress/_helpers/footer.rs b/crates/sqlbuild-core/src/build_progress/_helpers/footer.rs new file mode 100644 index 0000000000..f3601560cb --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/_helpers/footer.rs @@ -0,0 +1,344 @@ +//! Counts, warnings, failures and the skipped rollup of the build footer. + +use crate::build_progress::_helpers::error_rows::failure_error_block; +use crate::build_progress::_helpers::text::pad; +use crate::build_progress::constants::{ + DELTA_SUFFIX, ERROR, FAILED, PASS, PYTHON_NODE_NAME_WIDTH, SKIPPED, SKIPPED_ROLLUP_CAP, + SUCCESS, WARN, +}; +use crate::build_progress::models::{AuditResultText, BuildResultText, ErrorText, PythonNodeText}; +use crate::cli_text::models::CliStyle; + +/// Pass, warn, fail and skip counts of the footer. +#[derive(Default)] +pub(crate) struct FooterCounts { + pub(crate) pass: i64, + pub(crate) warn: i64, + pub(crate) fail: i64, + pub(crate) skip: i64, +} + +impl FooterCounts { + fn status(&mut self, status: &str, warned: bool) { + match status { + SUCCESS if warned => self.warn += 1, + SUCCESS => self.pass += 1, + FAILED => self.fail += 1, + SKIPPED => self.skip += 1, + _ => {} + } + } +} + +fn build_audits(result: &BuildResultText) -> impl Iterator { + result + .models + .iter() + .flat_map(|model| model.audits.iter()) + .chain(result.source_audits.iter()) + .chain(result.end_audits.iter()) +} + +pub(crate) fn count_results(result: &BuildResultText, python: &[PythonNodeText]) -> FooterCounts { + let mut counts: FooterCounts = FooterCounts::default(); + for model in &result.models { + counts.status(&model.status, !model.warnings.is_empty()); + } + for audit in build_audits(result) { + match audit.outcome.as_str() { + PASS => counts.pass += 1, + WARN => counts.warn += 1, + ERROR => counts.fail += 1, + _ => {} + } + } + for seed in &result.seeds { + counts.status(&seed.status, false); + } + for function in &result.functions { + counts.status(&function.status, false); + counts.warn += function.warnings.len() as i64; + } + for load in &result.loads { + counts.status(&load.status, !load.warnings.is_empty()); + } + for test in &result.tests { + if test.outcome == PASS { + counts.pass += 1; + } else { + counts.fail += 1; + } + } + for node in python { + counts.status(&node.status, !node.warnings.is_empty()); + } + counts +} + +/// One titled footer section that is shown only when it has entries. +struct Section { + title: String, + lines: Vec, +} + +impl Section { + fn new(title: String) -> Self { + Self { + title, + lines: Vec::new(), + } + } + + fn entry(&mut self, heading: String, body: impl IntoIterator) { + self.lines.push(heading); + self.lines.extend(body); + self.lines.push(String::new()); + } + + fn failure(&mut self, heading: String, error: &ErrorText, style: CliStyle) { + self.failure_with(heading, error, None, style); + } + + fn failure_with( + &mut self, + heading: String, + error: &ErrorText, + extra: Option, + style: CliStyle, + ) { + let mut body: Vec = Vec::new(); + if let Some(message) = &error.message { + body.extend(failure_error_block( + (error.code.as_deref(), message, error.help.as_deref()), + style, + )); + } + body.extend(extra); + self.entry(heading, body); + } + + fn finish(self) -> Vec { + if self.lines.is_empty() { + return Vec::new(); + } + let mut lines: Vec = vec![String::new(), self.title, String::new()]; + lines.extend(self.lines); + lines + } +} + +fn audit_warning(audit: &AuditResultText) -> String { + let name: String = match audit.attached_column_name.as_deref() { + Some(column) if !column.is_empty() => format!("{} ({column})", audit.audit_name), + _ => audit.audit_name.clone(), + }; + if audit.measurement { + return format!(" audit {name} measured {}", audit.measured_value); + } + let row_label: &str = if audit.row_count == 1 { "row" } else { "rows" }; + format!(" audit {name} returned {} {row_label}", audit.row_count) +} + +pub(crate) fn warning_details( + result: &BuildResultText, + python: &[PythonNodeText], + style: CliStyle, +) -> Vec { + let mut section: Section = Section::new(style.warning_strong("Warnings:")); + for model in &result.models { + let mut warnings: Vec = model + .audits + .iter() + .filter(|audit| audit.outcome == WARN) + .map(audit_warning) + .collect(); + warnings.extend( + model + .warnings + .iter() + .map(|warning| format!(" {warning}")), + ); + if !warnings.is_empty() { + section.entry(format!(" {}", model.name), warnings); + } + } + let labelled = result + .functions + .iter() + .map(|function| { + ( + format!("{} ({})", function.name, function.kind), + &function.warnings, + ) + }) + .chain(result.loads.iter().map(|load| { + ( + format!("{} (loader {})", load.source_name, load.loader_name), + &load.warnings, + ) + })) + .chain( + python + .iter() + .map(|node| (format!("{} ({})", node.name, node.kind), &node.warnings)), + ); + for (label, messages) in labelled { + if !messages.is_empty() { + section.entry( + format!(" {label}"), + messages.iter().map(|message| format!(" {message}")), + ); + } + } + section.finish() +} + +fn inspection_relation_message(relation: &str) -> String { + if relation.ends_with(DELTA_SUFFIX) { + return format!("delta table kept for inspection: {relation}"); + } + format!("staging table kept for inspection: {relation}") +} + +fn message_error(message: Option<&String>) -> ErrorText { + ErrorText { + message: message.cloned(), + ..ErrorText::default() + } +} + +pub(crate) fn failure_details(result: &BuildResultText, style: CliStyle) -> Vec { + let mut section: Section = Section::new(style.error_strong("Failures:")); + for load in result.loads.iter().filter(|load| load.status == FAILED) { + section.failure( + format!(" {} ({})", load.source_name, load.resource_kind), + &message_error(load.error_message.as_ref()), + style, + ); + } + for seed in result.seeds.iter().filter(|seed| seed.status == FAILED) { + section.failure(format!(" {} (seed)", seed.name), &seed.error, style); + } + for model in result.models.iter().filter(|model| model.status == FAILED) { + let phase: String = match model.failed_phase.as_deref() { + Some(phase) if !phase.is_empty() => format!(" ({phase})"), + _ => String::new(), + }; + section.failure_with( + format!(" {}{phase}", model.name), + &model.error, + model + .staging_relation + .as_deref() + .map(|relation| format!(" {}", inspection_relation_message(relation))), + style, + ); + } + for function in result + .functions + .iter() + .filter(|function| function.status == FAILED) + { + section.failure( + format!(" {} ({})", function.name, function.kind), + &function.error, + style, + ); + } + for test in result.tests.iter().filter(|test| test.outcome != PASS) { + section.failure(format!(" {} (test)", test.label), &test.error, style); + } + for audit in result.source_audits.iter().chain(result.end_audits.iter()) { + if audit.execution_error.is_none() { + continue; + } + let target: String = audit + .attached_target_name + .as_deref() + .map_or_else(String::new, |target| format!(" on {target}")); + section.failure( + format!(" {}{target} (audit)", audit.audit_name), + &message_error(audit.execution_error.as_ref()), + style, + ); + } + section.finish() +} + +pub(crate) fn python_failure_details(python: &[PythonNodeText], style: CliStyle) -> Vec { + let failed: Vec<&PythonNodeText> = python.iter().filter(|node| node.status == FAILED).collect(); + if failed.is_empty() { + return Vec::new(); + } + let mut lines: Vec = vec![ + String::new(), + style.error_strong("Python node failures:"), + String::new(), + ]; + for node in failed { + let message: &str = match node.error_message.as_deref() { + Some(message) if !message.is_empty() => message, + _ => "Python node failed", + }; + lines.push(format!( + " {}{} {}", + pad(&node.kind, 10), + pad(&node.name, PYTHON_NODE_NAME_WIDTH), + style.error(message) + )); + } + lines +} + +pub(crate) fn skipped_rollup( + result: &BuildResultText, + python: &[PythonNodeText], + style: CliStyle, +) -> Vec { + let names: Vec<&str> = result + .models + .iter() + .filter(|model| model.status == SKIPPED) + .map(|model| model.name.as_str()) + .chain( + result + .seeds + .iter() + .filter(|seed| seed.status == SKIPPED) + .map(|seed| seed.name.as_str()), + ) + .chain( + result + .functions + .iter() + .filter(|function| function.status == SKIPPED) + .map(|function| function.name.as_str()), + ) + .chain( + result + .loads + .iter() + .filter(|load| load.status == SKIPPED) + .map(|load| load.source_name.as_str()), + ) + .chain( + python + .iter() + .filter(|node| node.status == SKIPPED) + .map(|node| node.name.as_str()), + ) + .collect(); + if names.is_empty() { + return Vec::new(); + } + let visible: Vec<&str> = names.iter().take(SKIPPED_ROLLUP_CAP).copied().collect(); + let remaining: usize = names.len() - visible.len(); + let mut rendered: String = visible.join(", "); + if remaining > 0 { + rendered = format!("{rendered}, ... (+{remaining} more)"); + } + vec![ + String::new(), + style.muted(&format!("Skipped ({}): {rendered}", names.len())), + ] +} diff --git a/crates/sqlbuild-core/src/build_progress/_helpers/mod.rs b/crates/sqlbuild-core/src/build_progress/_helpers/mod.rs new file mode 100644 index 0000000000..fe214acff5 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod audits; +pub(crate) mod error_rows; +pub(crate) mod footer; +pub(crate) mod model_rows; +pub(crate) mod rows; +pub(crate) mod text; diff --git a/crates/sqlbuild-core/src/build_progress/_helpers/model_rows.rs b/crates/sqlbuild-core/src/build_progress/_helpers/model_rows.rs new file mode 100644 index 0000000000..152aa2def0 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/_helpers/model_rows.rs @@ -0,0 +1,248 @@ +//! The rows nested under a finished model row. + +use crate::build_progress::_helpers::audits::{AuditDisplayEntry, aggregate_audit_results}; +use crate::build_progress::_helpers::rows::{TopLevelRow, log_block, sql_block, top_level_line}; +use crate::build_progress::_helpers::text::{ + abbreviated_rows, audit_outcome_display, counter_text, execution_status_display, + format_duration, pad, skip_detail, test_outcome_display, truncate_name, +}; +use crate::build_progress::constants::{ + BATCH_INDENT, FAILED, HOOK_MAX_LABEL_WIDTH, HOOK_MIN_LABEL_WIDTH, HOOK_PHASE_WIDTH, + HOOK_TYPE_WIDTH, PASS, SKIPPED, TYPE_WIDTH, +}; +use crate::build_progress::models::{ + HookResultText, LifecycleEventText, ModelResultText, ModelRowFacts, ProgressLayout, + TestResultText, +}; +use crate::cli_text::models::CliStyle; +use crate::plan_display::main::format_no_input_rows::format_no_input_rows; +use crate::plan_display::main::format_waiting_on_empty_inputs::format_waiting_on_empty_inputs; +use crate::progress::main::format_expectation_name::format_expectation_name; + +/// The indent and name width of rows nested under the model row. +pub(crate) struct SubRows { + pub(crate) pad: String, + pub(crate) name_width: usize, + pub(crate) style: CliStyle, +} + +pub(crate) fn streaming_batch_summary(model: &ModelResultText) -> Option { + let batch_count: i64 = model.batch_count?; + let label: &str = if batch_count == 1 { "batch" } else { "batches" }; + let mut count_text: String = format!("{batch_count} {label}"); + if let Some(size) = &model.batch_size { + count_text = format!("{count_text} ({size})"); + } + let mut parts: Vec = vec![count_text]; + if let Some((start, end)) = &model.cursor_range + && model.empty_cursor_inputs.is_empty() + { + parts.push(format!("range {start} \u{2192} {end}")); + } + if let Some(rows) = model.rows_affected { + parts.push(abbreviated_rows(rows)); + } + if !model.empty_cursor_inputs.is_empty() && !model.waiting_on_empty_inputs { + parts.push(format_no_input_rows(&model.empty_cursor_inputs)); + } + if model.recovery_batch_count != 0 { + parts.push(format!("{} recovery", model.recovery_batch_count)); + } + if model.synthetic_completion_count != 0 { + parts.push(format!( + "{} synthetic coverage", + model.synthetic_completion_count + )); + } + if let Some(state) = &model.replay_requirement_state { + parts.push(format!("replay {state}")); + } + Some(format!("{BATCH_INDENT}{}", parts.join(" "))) +} + +pub(crate) fn empty_cursor_inputs(model: &ModelResultText) -> Option { + if model.empty_cursor_inputs.is_empty() || model.waiting_on_empty_inputs { + return None; + } + Some(format!( + "{BATCH_INDENT}{}", + format_no_input_rows(&model.empty_cursor_inputs) + )) +} + +fn hook_label_width(hooks: &[HookResultText]) -> usize { + let Some(longest) = hooks.iter().map(|hook| hook.label.chars().count()).max() else { + return HOOK_MIN_LABEL_WIDTH; + }; + HOOK_MIN_LABEL_WIDTH.max(longest.min(HOOK_MAX_LABEL_WIDTH)) +} + +pub(crate) fn hook_rows(hooks: &[HookResultText], pre_hook: bool, sub: &SubRows) -> String { + let width: usize = hook_label_width(hooks); + let label: &str = if pre_hook { "pre_hook" } else { "post_hook" }; + let mut text: String = String::new(); + for hook in hooks.iter().filter(|hook| hook.pre_hook == pre_hook) { + let detail: String = if hook.status == SKIPPED { + skip_detail(&hook.skip) + } else { + String::new() + }; + text.push_str(&format!( + "{}{}{}{} {}{detail}\n", + sub.pad, + sub.style.muted(&pad(label, HOOK_PHASE_WIDTH)), + sub.style.muted(&pad(&hook.hook_type, HOOK_TYPE_WIDTH)), + pad(&truncate_name(&hook.label, width), width), + sub.style.status(&execution_status_display(&hook.status)) + )); + } + text +} + +pub(crate) fn test_rows(tests: &[TestResultText], sub: &SubRows) -> String { + let style: CliStyle = sub.style; + let mut text: String = String::new(); + for test in tests { + text.push_str(&format!( + "{}{}{} {}\n", + sub.pad, + style.muted(&pad("test", TYPE_WIDTH)), + pad(&test.label, sub.name_width), + style.status(test_outcome_display(&test.outcome)) + )); + for step in &test.steps { + text.push_str(&format!( + "{} {}{} {}{}\n", + sub.pad, + style.muted(&pad("expect", TYPE_WIDTH - 2)), + pad(&format_expectation_name(&step.model_name), sub.name_width), + style.status(test_outcome_display(&step.outcome)), + step.detail + )); + } + } + text +} + +pub(crate) fn audit_row(entry: &AuditDisplayEntry, sub: &SubRows) -> String { + let mut detail: String = String::new(); + if entry.reused { + detail = " proof reused".to_owned(); + } + if entry.outcome != PASS && entry.total_row_count > 0 { + let label: &str = if entry.total_row_count == 1 { + "row" + } else { + "rows" + }; + detail = format!(" {} {label}", entry.total_row_count); + } + if entry.batch_total > 1 { + detail = format!(" {}/{}{detail}", entry.batch_pass, entry.batch_total); + } + format!( + "{}{}{} {}{detail}\n", + sub.pad, + sub.style.muted(&pad(entry.label, TYPE_WIDTH)), + pad( + &truncate_name(&entry.display_name, sub.name_width), + sub.name_width + ), + sub.style.status(&audit_outcome_display(entry.outcome)) + ) +} + +/// The model row itself, with its annotation, phase or skip detail. +pub(crate) fn model_line( + (model, facts): (&ModelResultText, &ModelRowFacts), + counter: usize, + layout: ProgressLayout, + style: CliStyle, +) -> String { + let mut name: String = model.name.clone(); + if !facts.annotation.is_empty() { + name = format!("{name} ({})", facts.annotation); + } + let mut duration: String = format_duration(model.duration_ms); + let mut detail: String = String::new(); + if model.status == FAILED + && let Some(phase) = &model.failed_phase + { + detail = format!(" {phase}"); + } else if model.status == SKIPPED { + duration = String::new(); + detail = skip_detail(&model.skip); + } + top_level_line( + &TopLevelRow { + counter: &counter_text(counter, layout.total), + resource_type: &facts.display_type, + name: &truncate_name(&name, layout.name_width), + status: &execution_status_display(&model.status), + duration: &duration, + detail: &detail, + }, + layout, + style, + ) +} + +/// Lifecycle SQL and log blocks, falling back to the planned DDL. +pub(crate) fn verbose_blocks( + model: &ModelResultText, + facts: &ModelRowFacts, + style: CliStyle, +) -> String { + let fallback: Vec = facts + .logical_ddl + .iter() + .map(|ddl| LifecycleEventText::Sql(ddl.clone())) + .collect(); + let events: &[LifecycleEventText] = if model.events.is_empty() { + &fallback + } else { + &model.events + }; + events + .iter() + .map(|event| match event { + LifecycleEventText::Sql(sql) => sql_block(sql, style), + LifecycleEventText::Log(message) => log_block(message, style), + }) + .collect() +} + +/// The batch summary or empty-input note, and the waiting-on-empty-inputs warning. +pub(crate) fn batch_lines(model: &ModelResultText, style: CliStyle) -> String { + let mut text: String = String::new(); + if let Some(line) = streaming_batch_summary(model).or_else(|| empty_cursor_inputs(model)) { + text.push_str(&format!("{}\n", style.muted(&line))); + } + if model.waiting_on_empty_inputs && !model.empty_cursor_inputs.is_empty() { + let line: String = format!( + "{BATCH_INDENT}{}", + format_waiting_on_empty_inputs(&model.empty_cursor_inputs) + ); + text.push_str(&format!("{}\n", style.warning(&line))); + } + text +} + +/// Pre-hooks, tests, audits with their verbose SQL, then post-hooks. +pub(crate) fn nested_rows( + model: &ModelResultText, + tests: &[TestResultText], + sub: &SubRows, + verbose: bool, +) -> String { + let mut text: String = hook_rows(&model.hooks, true, sub); + text.push_str(&test_rows(tests, sub)); + for entry in aggregate_audit_results(&model.audits) { + text.push_str(&audit_row(&entry, sub)); + if verbose && let Some(sql) = &entry.executed_sql { + text.push_str(&sql_block(sql, sub.style)); + } + } + text.push_str(&hook_rows(&model.hooks, false, sub)); + text +} diff --git a/crates/sqlbuild-core/src/build_progress/_helpers/rows.rs b/crates/sqlbuild-core/src/build_progress/_helpers/rows.rs new file mode 100644 index 0000000000..8f33a80252 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/_helpers/rows.rs @@ -0,0 +1,82 @@ +//! Row pieces shared by the persistent build progress lines. + +use crate::build_progress::_helpers::text::{counter_width, display_sql, pad}; +use crate::build_progress::constants::{LOG_CONTINUATION, LOG_LABEL, TYPE_WIDTH}; +use crate::build_progress::models::{LifecycleEventText, ProgressLayout}; +use crate::cli_text::main::format_status_cell::format_status_cell; +use crate::cli_text::models::CliStyle; +use crate::text::main::python_splitlines::python_splitlines; + +/// The width of the ` N/M ` prefix every top-level row starts with. +pub(crate) fn prefix_width(layout: ProgressLayout) -> usize { + 2 + counter_width(layout.total) + 2 +} + +/// The cells of one top-level result row. +pub(crate) struct TopLevelRow<'a> { + pub(crate) counter: &'a str, + pub(crate) resource_type: &'a str, + pub(crate) name: &'a str, + pub(crate) status: &'a str, + pub(crate) duration: &'a str, + pub(crate) detail: &'a str, +} + +pub(crate) fn top_level_line( + row: &TopLevelRow<'_>, + layout: ProgressLayout, + style: CliStyle, +) -> String { + let duration: String = if row.duration.is_empty() { + String::new() + } else { + style.muted(row.duration) + }; + format!( + " {} {}{} {} {duration}{}\n", + row.counter, + style.muted(&pad(row.resource_type, TYPE_WIDTH)), + pad(row.name, layout.name_width), + format_status_cell(style, row.status, 6), + row.detail + ) +} + +pub(crate) fn sql_block(sql: &str, style: CliStyle) -> String { + let mut text: String = "\n".to_owned(); + for line in display_sql(sql).split('\n') { + text.push_str(&format!("{}\n", style.muted(&format!(" {line}")))); + } + text.push('\n'); + text +} + +pub(crate) fn log_block(message: &str, style: CliStyle) -> String { + let lines: Vec<&str> = python_splitlines(message); + let first: &str = lines.first().copied().unwrap_or(""); + let mut text: String = format!("\n{}{}\n", style.label(LOG_LABEL), style.muted(first)); + for line in lines.iter().skip(1) { + text.push_str(&format!( + "{}\n", + style.muted(&format!("{LOG_CONTINUATION}{line}")) + )); + } + text +} + +/// Log blocks, and SQL blocks when verbose, of a lifecycle event list. +pub(crate) fn event_blocks( + events: &[LifecycleEventText], + verbose: bool, + style: CliStyle, +) -> String { + let mut text: String = String::new(); + for event in events { + match event { + LifecycleEventText::Log(message) => text.push_str(&log_block(message, style)), + LifecycleEventText::Sql(sql) if verbose => text.push_str(&sql_block(sql, style)), + LifecycleEventText::Sql(_) => {} + } + } + text +} diff --git a/crates/sqlbuild-core/src/build_progress/_helpers/text.rs b/crates/sqlbuild-core/src/build_progress/_helpers/text.rs new file mode 100644 index 0000000000..ce726f2a01 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/_helpers/text.rs @@ -0,0 +1,142 @@ +//! Python formatting idioms and small labels shared by build progress rows. + +use crate::build_progress::constants::{ + ERROR, FAILED, INSUFFICIENT, PASS, ROW_COUNT_ABBREVIATION_THRESHOLD, ROW_COUNT_BILLION, + ROW_COUNT_MILLION, ROW_COUNT_THOUSAND, SECONDS_PER_MINUTE, SKIPPED, SUCCESS, WARN, +}; +use crate::build_progress::models::SkipText; +use crate::text::main::python_isspace::python_isspace; + +/// `f"{text: String { + let length: usize = text.chars().count(); + format!("{text}{}", " ".repeat(width.saturating_sub(length))) +} + +/// `f"{text:>width}"`: right-aligned, padded by character count. +pub(crate) fn pad_left(text: &str, width: usize) -> String { + let length: usize = text.chars().count(); + format!("{}{text}", " ".repeat(width.saturating_sub(length))) +} + +/// `text[:count]` by character. +pub(crate) fn head(text: &str, count: usize) -> String { + text.chars().take(count).collect() +} + +/// `f"{value:,}"`: the integer with comma-separated thousands. +pub(crate) fn group_thousands(value: i64) -> String { + let digits: String = value.unsigned_abs().to_string(); + let mut grouped: String = String::new(); + for (index, digit) in digits.chars().enumerate() { + if index > 0 && (digits.len() - index).is_multiple_of(3) { + grouped.push(','); + } + grouped.push(digit); + } + if value < 0 { + format!("-{grouped}") + } else { + grouped + } +} + +/// The name cut to `width` characters with a trailing ellipsis. +pub(crate) fn truncate_name(name: &str, width: usize) -> String { + if name.chars().count() <= width { + return name.to_owned(); + } + format!("{}...", head(name, width.saturating_sub(3))) +} + +/// `"{counter}/{total}"` right-aligned to twice the total's digits plus one. +pub(crate) fn counter_text(counter: usize, total: usize) -> String { + pad_left(&format!("{counter}/{total}"), counter_width(total)) +} + +pub(crate) fn counter_width(total: usize) -> usize { + total.to_string().len() * 2 + 1 +} + +/// Seconds with two decimals under a minute, minutes and seconds otherwise. +pub(crate) fn format_duration(duration_ms: Option) -> String { + let Some(duration_ms) = duration_ms else { + return String::new(); + }; + let seconds: f64 = duration_ms / 1000.0; + if seconds < SECONDS_PER_MINUTE { + return format!("{seconds:.2}s"); + } + let minutes: i64 = (seconds / SECONDS_PER_MINUTE).floor() as i64; + let remaining: f64 = seconds - minutes as f64 * SECONDS_PER_MINUTE; + format!("{minutes}m{remaining:.1}s") +} + +/// The OK, FAIL or SKIP word of an execution status. +pub(crate) fn execution_status_display(status: &str) -> String { + match status { + SUCCESS => "OK".to_owned(), + FAILED => "FAIL".to_owned(), + SKIPPED => "SKIP".to_owned(), + other => other.to_owned(), + } +} + +/// The status word of an audit outcome. +pub(crate) fn audit_outcome_display(outcome: &str) -> String { + match outcome { + PASS => "PASS".to_owned(), + WARN => "WARN".to_owned(), + ERROR => "FAIL".to_owned(), + INSUFFICIENT => "INSUFFICIENT".to_owned(), + other => other.to_owned(), + } +} + +/// The status word of a SQL test outcome. +pub(crate) fn test_outcome_display(outcome: &str) -> &'static str { + match outcome { + PASS => "PASS", + ERROR => "ERROR", + _ => "FAIL", + } +} + +/// ` skip: ` for a skipped row. +pub(crate) fn skip_detail(skip: &SkipText) -> String { + let label: String = skip + .mode + .as_ref() + .map_or_else(|| "skip".to_owned(), |mode| format!("{mode} skip")); + match skip.reason.as_deref() { + Some(reason) if !reason.is_empty() => format!(" {label}: {reason}"), + _ => format!(" {label}"), + } +} + +/// The SQL with trailing whitespace removed and a terminating semicolon. +pub(crate) fn display_sql(sql: &str) -> String { + let stripped: &str = sql.trim_end_matches(python_isspace); + if stripped.is_empty() { + return sql.to_owned(); + } + if stripped.ends_with(';') { + return stripped.to_owned(); + } + format!("{stripped};") +} + +/// A row count with K/M/B abbreviation from ten thousand rows. +pub(crate) fn abbreviated_rows(count: i64) -> String { + let label: &str = if count == 1 { "row" } else { "rows" }; + if count >= ROW_COUNT_BILLION { + return format!("{:.1}B {label}", count as f64 / ROW_COUNT_BILLION as f64); + } + if count >= ROW_COUNT_MILLION { + return format!("{:.1}M {label}", count as f64 / ROW_COUNT_MILLION as f64); + } + if count >= ROW_COUNT_ABBREVIATION_THRESHOLD { + return format!("{:.1}K {label}", count as f64 / ROW_COUNT_THOUSAND as f64); + } + format!("{} {label}", group_thousands(count)) +} diff --git a/crates/sqlbuild-core/src/build_progress/constants.rs b/crates/sqlbuild-core/src/build_progress/constants.rs new file mode 100644 index 0000000000..648027a09d --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/constants.rs @@ -0,0 +1,37 @@ +//! Column widths, limits and status values of build progress output. + +pub(crate) const TYPE_WIDTH: usize = 10; +pub(crate) const MIN_NAME_WIDTH: usize = 20; +pub(crate) const NAME_PADDING: usize = 2; +pub(crate) const SUB_INDENT: usize = 2; +pub(crate) const TREE_CHILD_NAME_INDENT: usize = SUB_INDENT + 4; +pub(crate) const HOOK_PHASE_WIDTH: usize = 10; +pub(crate) const HOOK_TYPE_WIDTH: usize = 8; +pub(crate) const HOOK_MIN_LABEL_WIDTH: usize = 24; +pub(crate) const HOOK_MAX_LABEL_WIDTH: usize = 56; +pub(crate) const SKIPPED_ROLLUP_CAP: usize = 20; +pub(crate) const ROW_COUNT_BILLION: i64 = 1_000_000_000; +pub(crate) const ROW_COUNT_MILLION: i64 = 1_000_000; +pub(crate) const ROW_COUNT_THOUSAND: i64 = 1_000; +pub(crate) const ROW_COUNT_ABBREVIATION_THRESHOLD: i64 = 10_000; +pub(crate) const SECONDS_PER_MINUTE: f64 = 60.0; +pub(crate) const SCHEDULER_DIAGNOSTIC_LIMIT: usize = 50; +pub(crate) const QUERY_DIAGNOSTIC_LIMIT: usize = 25; +pub(crate) const SPINNER_FRAMES: [&str; 10] = [ + "\u{280b}", "\u{2819}", "\u{2839}", "\u{2838}", "\u{283c}", "\u{2834}", "\u{2826}", "\u{2827}", + "\u{2807}", "\u{280f}", +]; +pub(crate) const BATCH_INDENT: &str = " "; +pub(crate) const LOG_LABEL: &str = " log "; +pub(crate) const LOG_CONTINUATION: &str = " "; +pub(crate) const PYTHON_NODE_NAME_WIDTH: usize = 50; +pub(crate) const DELTA_SUFFIX: &str = "__delta"; + +pub(crate) const SUCCESS: &str = "success"; +pub(crate) const FAILED: &str = "failed"; +pub(crate) const SKIPPED: &str = "skipped"; +pub(crate) const PASS: &str = "pass"; +pub(crate) const WARN: &str = "warn"; +pub(crate) const ERROR: &str = "error"; +pub(crate) const INSUFFICIENT: &str = "insufficient"; +pub(crate) const DELTA_AND_FINAL: &str = "delta_and_final"; diff --git a/crates/sqlbuild-core/src/build_progress/main/format_build_footer.rs b/crates/sqlbuild-core/src/build_progress/main/format_build_footer.rs new file mode 100644 index 0000000000..97255700f9 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/format_build_footer.rs @@ -0,0 +1,57 @@ +//! The completion line, warnings, failures and skipped rollup printed after a build. + +use crate::build_progress::_helpers::footer::{ + FooterCounts, count_results, failure_details, python_failure_details, skipped_rollup, + warning_details, +}; +use crate::build_progress::constants::FAILED; +use crate::build_progress::models::{BuildResultText, PythonNodeText}; +use crate::cli_text::main::format_completion_line::format_completion_line; +use crate::cli_text::main::format_summary_footer::format_summary_footer; +use crate::cli_text::models::{CliStyle, CompletionState, SummaryCount}; + +fn count(label: &str, value: i64) -> SummaryCount { + SummaryCount { + label: label.to_owned(), + value, + } +} + +/// The build footer for `result` and the Python node results run alongside it. +#[must_use] +pub fn format_build_footer( + result: &BuildResultText, + python: &[PythonNodeText], + elapsed: f64, + style: CliStyle, +) -> String { + let counts: FooterCounts = count_results(result, python); + let insufficient: i64 = result.insufficient_count; + let total: i64 = counts.pass + counts.warn + counts.fail + counts.skip; + let summary: String = format_summary_footer( + style, + &[ + count("PASS", counts.pass), + count("WARN", counts.warn), + count("FAIL", counts.fail), + count("INSUFFICIENT", insufficient), + count("SKIP", counts.skip), + count("TOTAL", total + insufficient), + ], + Some(&format!("{elapsed:.2}s")), + ); + let python_failed: bool = python.iter().any(|node| node.status == FAILED); + let (state, label) = if result.failed || python_failed { + (CompletionState::Fail, "Completed with errors") + } else if result.warning_count > 0 || counts.warn > 0 { + (CompletionState::Warn, "Completed with warnings") + } else { + (CompletionState::Ok, "Completed successfully") + }; + let mut lines: Vec = warning_details(result, python, style); + lines.push(format_completion_line(style, state, label, Some(&summary))); + lines.extend(failure_details(result, style)); + lines.extend(python_failure_details(python, style)); + lines.extend(skipped_rollup(result, python, style)); + lines.join("\n") +} diff --git a/crates/sqlbuild-core/src/build_progress/main/mod.rs b/crates/sqlbuild-core/src/build_progress/main/mod.rs new file mode 100644 index 0000000000..09e6110e07 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/mod.rs @@ -0,0 +1,9 @@ +pub mod format_build_footer; +pub mod progress_name_width; +pub mod query_diagnostic_message; +pub mod render_build_spinner_line; +pub mod render_failed_test_result; +pub mod render_model_result; +pub mod render_resource_result; +pub mod runtime_diagnostic_summary; +pub mod scheduler_diagnostic_message; diff --git a/crates/sqlbuild-core/src/build_progress/main/progress_name_width.rs b/crates/sqlbuild-core/src/build_progress/main/progress_name_width.rs new file mode 100644 index 0000000000..973ec3533b --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/progress_name_width.rs @@ -0,0 +1,16 @@ +//! The shared name column width of build progress rows. + +use crate::build_progress::constants::{MIN_NAME_WIDTH, NAME_PADDING, TREE_CHILD_NAME_INDENT}; +use crate::build_progress::models::NameWidthInputs; +use crate::progress::main::format_expectation_name::format_expectation_name; + +/// The longest row name plus padding, so tree children still align with top-level statuses. +#[must_use] +pub fn progress_name_width(inputs: &NameWidthInputs) -> usize { + let top_level = inputs.names.iter().map(|name| name.chars().count()); + let children = inputs.tree_child_names.iter().map(|name| { + format_expectation_name(name).chars().count() + TREE_CHILD_NAME_INDENT - NAME_PADDING + }); + let longest: usize = top_level.chain(children).max().unwrap_or(0); + (longest + NAME_PADDING).max(MIN_NAME_WIDTH) +} diff --git a/crates/sqlbuild-core/src/build_progress/main/query_diagnostic_message.rs b/crates/sqlbuild-core/src/build_progress/main/query_diagnostic_message.rs new file mode 100644 index 0000000000..31259bccfc --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/query_diagnostic_message.rs @@ -0,0 +1,20 @@ +//! The diagnostic line of one completed warehouse statement. + +use crate::build_progress::models::StatementText; + +/// The resource, phase, query identity, status and elapsed time of a statement. +#[must_use] +pub fn query_diagnostic_message(statement: &StatementText) -> String { + let query_id: &str = match statement.query_id.as_deref() { + Some(query_id) if !query_id.is_empty() => query_id, + _ => "query ID unavailable", + }; + format!( + "Query {} {} phase={} {query_id} {} {:.2}s", + statement.resource_type, + statement.resource_name, + statement.phase, + statement.status, + statement.elapsed_seconds + ) +} diff --git a/crates/sqlbuild-core/src/build_progress/main/render_build_spinner_line.rs b/crates/sqlbuild-core/src/build_progress/main/render_build_spinner_line.rs new file mode 100644 index 0000000000..7a1852f4a5 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/render_build_spinner_line.rs @@ -0,0 +1,32 @@ +//! The live spinner row of the running build node. + +use crate::build_progress::_helpers::text::{counter_text, head, pad, truncate_name}; +use crate::build_progress::constants::{MIN_NAME_WIDTH, SPINNER_FRAMES, TYPE_WIDTH}; +use crate::build_progress::models::{ProgressLayout, SpinnerRow}; +use crate::cli_text::models::CliStyle; + +/// The cleared line with the position, kind, name, message and spinner frame `frame`. +#[must_use] +pub fn render_build_spinner_line( + row: &SpinnerRow, + frame: usize, + layout: ProgressLayout, + style: CliStyle, +) -> String { + let mut name: String = truncate_name(&row.name, layout.name_width); + if !row.message.is_empty() { + name = format!("{name} {}", row.message); + } + let mut plain: String = format!( + " {} {}{} ", + counter_text(row.position, layout.total), + pad(&row.resource_kind, TYPE_WIDTH), + pad(&name, layout.name_width) + ); + let max_width: i64 = (layout.columns as i64 - 2).max(MIN_NAME_WIDTH as i64); + if plain.chars().count() as i64 > max_width { + plain = format!("{}... ", head(&plain, (max_width - 4).max(0) as usize)); + } + let spinner: &str = SPINNER_FRAMES[frame % SPINNER_FRAMES.len()]; + format!("\r\x1b[K{plain}{}", style.status(spinner)) +} diff --git a/crates/sqlbuild-core/src/build_progress/main/render_failed_test_result.rs b/crates/sqlbuild-core/src/build_progress/main/render_failed_test_result.rs new file mode 100644 index 0000000000..375d60934b --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/render_failed_test_result.rs @@ -0,0 +1,54 @@ +//! A failing SQL test row written inline, with its expectations as tree children. + +use crate::build_progress::_helpers::error_rows::error_detail; +use crate::build_progress::_helpers::rows::prefix_width; +use crate::build_progress::_helpers::text::{ + counter_width, pad, test_outcome_display, truncate_name, +}; +use crate::build_progress::constants::{SUB_INDENT, TREE_CHILD_NAME_INDENT, TYPE_WIDTH}; +use crate::build_progress::models::{ProgressLayout, TestResultText}; +use crate::cli_text::main::format_tree_connector::format_tree_connector; +use crate::cli_text::models::CliStyle; +use crate::progress::main::format_expectation_name::format_expectation_name; + +/// The test row without a position, its expectation rows and its error. +#[must_use] +pub fn render_failed_test_result( + test: &TestResultText, + layout: ProgressLayout, + style: CliStyle, +) -> String { + let mut text: String = format!( + " {} {}{} {}\n", + " ".repeat(counter_width(layout.total)), + style.muted(&pad("test", TYPE_WIDTH)), + pad( + &truncate_name(&test.label, layout.name_width), + layout.name_width + ), + style.status(test_outcome_display(&test.outcome)) + ); + let error: Option<&str> = test.error.message.as_deref().filter(|m| !m.is_empty()); + let expectation_pad: String = " ".repeat(prefix_width(layout) + SUB_INDENT + 2); + let child_width: usize = layout.name_width.saturating_sub(TREE_CHILD_NAME_INDENT); + for (index, step) in test.steps.iter().enumerate() { + let last: bool = index + 1 == test.steps.len() && error.is_none(); + text.push_str(&format!( + "{expectation_pad}{} {}{} {}{}\n", + format_tree_connector(style, last), + style.muted(&pad("expect", TYPE_WIDTH - 2)), + pad(&format_expectation_name(&step.model_name), child_width), + style.status(test_outcome_display(&step.outcome)), + step.detail + )); + } + if let Some(message) = error { + text.push_str(&error_detail( + (&test.error, message), + (prefix_width(layout), layout.columns), + true, + style, + )); + } + text +} diff --git a/crates/sqlbuild-core/src/build_progress/main/render_model_result.rs b/crates/sqlbuild-core/src/build_progress/main/render_model_result.rs new file mode 100644 index 0000000000..62cb034bce --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/render_model_result.rs @@ -0,0 +1,44 @@ +//! The persistent row of a finished model with its batch, hook, test and audit rows. + +use crate::build_progress::_helpers::error_rows::error_detail; +use crate::build_progress::_helpers::model_rows::{ + SubRows, batch_lines, model_line, nested_rows, verbose_blocks, +}; +use crate::build_progress::_helpers::rows::prefix_width; +use crate::build_progress::constants::{FAILED, SUB_INDENT}; +use crate::build_progress::models::{ + ModelResultText, ModelRowFacts, ProgressLayout, TestResultText, +}; +use crate::cli_text::models::CliStyle; + +/// The model row at `counter` and every row nested under it. +#[must_use] +pub fn render_model_result( + (model, facts, tests): (&ModelResultText, &ModelRowFacts, &[TestResultText]), + counter: usize, + layout: ProgressLayout, + style: CliStyle, +) -> String { + let mut text: String = model_line((model, facts), counter, layout, style); + if layout.verbose { + text.push_str(&verbose_blocks(model, facts, style)); + } + text.push_str(&batch_lines(model, style)); + let sub: SubRows = SubRows { + pad: " ".repeat(prefix_width(layout) + SUB_INDENT), + name_width: layout.name_width.saturating_sub(SUB_INDENT), + style, + }; + text.push_str(&nested_rows(model, tests, &sub, layout.verbose)); + if model.status == FAILED + && let Some(message) = model.error.message.as_deref().filter(|m| !m.is_empty()) + { + text.push_str(&error_detail( + (&model.error, message), + (prefix_width(layout), layout.columns), + false, + style, + )); + } + text +} diff --git a/crates/sqlbuild-core/src/build_progress/main/render_resource_result.rs b/crates/sqlbuild-core/src/build_progress/main/render_resource_result.rs new file mode 100644 index 0000000000..6abf363972 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/render_resource_result.rs @@ -0,0 +1,82 @@ +//! The persistent row of a finished seed, function or source load. + +use crate::build_progress::_helpers::error_rows::error_detail; +use crate::build_progress::_helpers::rows::{ + TopLevelRow, event_blocks, prefix_width, top_level_line, +}; +use crate::build_progress::_helpers::text::{ + counter_text, execution_status_display, format_duration, group_thousands, skip_detail, + truncate_name, +}; +use crate::build_progress::constants::{FAILED, SKIPPED}; +use crate::build_progress::models::{ErrorText, ProgressLayout, ResourceResult}; +use crate::cli_text::models::CliStyle; + +/// The row at `counter`, its log and SQL blocks, and its inline error. +#[must_use] +pub fn render_resource_result( + result: ResourceResult<'_>, + counter: usize, + layout: ProgressLayout, + style: CliStyle, +) -> String { + let counter: String = counter_text(counter, layout.total); + match result { + ResourceResult::Node(node) => { + let mut text: String = top_level_line( + &TopLevelRow { + counter: &counter, + resource_type: &node.kind, + name: &truncate_name(&node.name, layout.name_width), + status: &execution_status_display(&node.status), + duration: &format_duration(node.duration_ms), + detail: "", + }, + layout, + style, + ); + if node.status == FAILED + && let Some(message) = node.error.message.as_deref().filter(|m| !m.is_empty()) + { + text.push_str(&error_detail( + (&node.error, message), + (prefix_width(layout), layout.columns), + false, + style, + )); + } + text + } + ResourceResult::Load(load) => { + let detail: String = if load.status == SKIPPED { + skip_detail(&load.skip) + } else { + format!(" rows={}", group_thousands(load.rows_loaded)) + }; + let mut text: String = top_level_line( + &TopLevelRow { + counter: &counter, + resource_type: &load.resource_kind, + name: &truncate_name(&load.source_name, layout.name_width), + status: &execution_status_display(&load.status), + duration: &format_duration(load.duration_ms), + detail: &detail, + }, + layout, + style, + ); + text.push_str(&event_blocks(&load.events, layout.verbose, style)); + if load.status == FAILED + && let Some(message) = load.error_message.as_deref().filter(|m| !m.is_empty()) + { + text.push_str(&error_detail( + (&ErrorText::default(), message), + (prefix_width(layout), layout.columns), + false, + style, + )); + } + text + } + } +} diff --git a/crates/sqlbuild-core/src/build_progress/main/runtime_diagnostic_summary.rs b/crates/sqlbuild-core/src/build_progress/main/runtime_diagnostic_summary.rs new file mode 100644 index 0000000000..bf42ad0e75 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/runtime_diagnostic_summary.rs @@ -0,0 +1,28 @@ +//! The closing lines for scheduler and query diagnostics omitted past their limits. + +use crate::build_progress::constants::{QUERY_DIAGNOSTIC_LIMIT, SCHEDULER_DIAGNOSTIC_LIMIT}; +use crate::build_progress::main::query_diagnostic_message::query_diagnostic_message; +use crate::build_progress::main::scheduler_diagnostic_message::scheduler_diagnostic_message; +use crate::build_progress::models::{SchedulerStateText, StatementText}; + +/// One line per diagnostic kind that omitted entries, naming the last omitted one. +#[must_use] +pub fn runtime_diagnostic_summary( + scheduler: Option<(usize, SchedulerStateText)>, + query: Option<(usize, &StatementText)>, +) -> Vec { + let mut lines: Vec = Vec::new(); + if let Some((omitted, state)) = scheduler.filter(|(omitted, _)| *omitted > 0) { + lines.push(format!( + "Scheduler diagnostics {omitted} transitions omitted after {SCHEDULER_DIAGNOSTIC_LIMIT}; final state: {}", + scheduler_diagnostic_message(state) + )); + } + if let Some((omitted, statement)) = query.filter(|(omitted, _)| *omitted > 0) { + lines.push(format!( + "Query diagnostics {omitted} statements omitted after {QUERY_DIAGNOSTIC_LIMIT}; final query: {}", + query_diagnostic_message(statement) + )); + } + lines +} diff --git a/crates/sqlbuild-core/src/build_progress/main/scheduler_diagnostic_message.rs b/crates/sqlbuild-core/src/build_progress/main/scheduler_diagnostic_message.rs new file mode 100644 index 0000000000..ecb2b673af --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/main/scheduler_diagnostic_message.rs @@ -0,0 +1,24 @@ +//! The diagnostic line of one concurrent scheduler state transition. + +use crate::build_progress::models::SchedulerStateText; + +/// Running, ready, waiting and aborted counts against the concurrency limit. +#[must_use] +pub fn scheduler_diagnostic_message(state: SchedulerStateText) -> String { + let counts: String = format!( + "Scheduler {} running, {} ready, {} waiting on dependencies", + state.running, state.ready, state.waiting + ); + if state.aborted != 0 { + return format!( + "{counts}, {} aborted after stop, limit {}", + state.aborted, state.limit + ); + } + let suffix: &str = if state.running < state.limit && state.ready == 0 && state.waiting > 0 { + " (DAG frontier constrained)" + } else { + "" + }; + format!("{counts}, limit {}{suffix}", state.limit) +} diff --git a/crates/sqlbuild-core/src/build_progress/mod.rs b/crates/sqlbuild-core/src/build_progress/mod.rs new file mode 100644 index 0000000000..ed82abaa70 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/mod.rs @@ -0,0 +1,9 @@ +//! Live progress rows and the completion footer of `sqb build`. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/build_progress/models.rs b/crates/sqlbuild-core/src/build_progress/models.rs new file mode 100644 index 0000000000..7ba6a5c391 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/models.rs @@ -0,0 +1,211 @@ +//! Plain data of executor results as build progress output renders them. + +/// The coded error recorded on one result. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ErrorText { + pub code: Option, + pub message: Option, + pub help: Option, +} + +/// Why a result was skipped. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SkipText { + pub mode: Option, + pub reason: Option, +} + +/// One lifecycle event of a model or load run. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum LifecycleEventText { + Sql(String), + Log(String), +} + +/// One audit result. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AuditResultText { + pub audit_name: String, + pub attached_column_name: Option, + pub attached_target_name: Option, + pub run_scope_phase: String, + pub outcome: String, + pub row_count: i64, + pub reused: bool, + pub executed_sql: Option, + /// The audit is evaluated as a measurement instead of violation rows. + pub measurement: bool, + /// `str(measured_value)`. + pub measured_value: String, + pub execution_error: Option, +} + +/// One pre- or post-hook result. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct HookResultText { + pub pre_hook: bool, + pub hook_type: String, + pub label: String, + pub status: String, + pub skip: SkipText, +} + +/// One model result. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ModelResultText { + pub name: String, + pub status: String, + pub failed_phase: Option, + pub staging_relation: Option, + pub duration_ms: Option, + pub batch_count: Option, + pub batch_size: Option, + pub rows_affected: Option, + /// The displayed first and inclusive last cursor bound, when both are known. + pub cursor_range: Option<(String, String)>, + pub empty_cursor_inputs: Vec, + pub waiting_on_empty_inputs: bool, + pub recovery_batch_count: i64, + pub synthetic_completion_count: i64, + pub replay_requirement_state: Option, + pub audits: Vec, + pub warnings: Vec, + pub events: Vec, + pub hooks: Vec, + pub skip: SkipText, + pub error: ErrorText, +} + +/// One seed or function result. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct NodeResultText { + pub name: String, + /// `seed`, or the function kind. + pub kind: String, + pub status: String, + pub duration_ms: Option, + pub error: ErrorText, + pub warnings: Vec, +} + +/// One source load result. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct LoadRowText { + pub source_name: String, + pub loader_name: String, + pub resource_kind: String, + pub status: String, + pub rows_loaded: i64, + pub duration_ms: Option, + pub events: Vec, + pub skip: SkipText, + pub error_message: Option, + pub warnings: Vec, +} + +/// One expectation step of a SQL test. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TestStepText { + pub model_name: String, + pub outcome: String, + pub detail: String, +} + +/// One SQL test result. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TestResultText { + /// The test name with its parameter label. + pub label: String, + pub outcome: String, + pub steps: Vec, + pub error: ErrorText, +} + +/// One Python node result. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PythonNodeText { + pub name: String, + pub kind: String, + pub status: String, + pub error_message: Option, + pub warnings: Vec, +} + +/// Every result of one build run. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct BuildResultText { + pub failed: bool, + pub warning_count: i64, + pub insufficient_count: i64, + pub models: Vec, + pub seeds: Vec, + pub functions: Vec, + pub loads: Vec, + pub tests: Vec, + pub source_audits: Vec, + pub end_audits: Vec, +} + +/// The plan facts of the model a row reports. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModelRowFacts { + pub display_type: String, + pub annotation: String, + pub logical_ddl: Option, +} + +/// Fixed geometry of the progress rows of one run. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ProgressLayout { + pub total: usize, + pub name_width: usize, + pub columns: usize, + pub verbose: bool, +} + +/// One concurrent scheduler state transition. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct SchedulerStateText { + pub running: i64, + pub ready: i64, + pub waiting: i64, + pub limit: i64, + pub aborted: i64, +} + +/// One completed warehouse statement. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct StatementText { + pub query_id: Option, + pub status: String, + pub elapsed_seconds: f64, + pub resource_type: String, + pub resource_name: String, + pub phase: String, +} + +/// The node shown on the live spinner row. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SpinnerRow { + /// The 1-based position the running node will complete at. + pub position: usize, + pub resource_kind: String, + pub name: String, + pub message: String, +} + +/// Names that set the shared name column width. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct NameWidthInputs { + /// Top-level row names, with model annotations. + pub names: Vec, + /// Expectation rows drawn as tree children. + pub tree_child_names: Vec, +} + +/// A finished seed or function, or a source load. +#[derive(Clone, Copy, Debug)] +pub enum ResourceResult<'a> { + Node(&'a NodeResultText), + Load(&'a LoadRowText), +} diff --git a/crates/sqlbuild-core/src/build_progress/tests.rs b/crates/sqlbuild-core/src/build_progress/tests.rs new file mode 100644 index 0000000000..58eafc32fe --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_build_progress.rs"] +mod build_progress; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/build_progress/tests/helpers.rs b/crates/sqlbuild-core/src/build_progress/tests/helpers.rs new file mode 100644 index 0000000000..68ea5bff0f --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/tests/helpers.rs @@ -0,0 +1,57 @@ +use crate::build_progress::models::{ + AuditResultText, ErrorText, ModelResultText, ProgressLayout, TestResultText, TestStepText, +}; + +pub(super) fn layout(verbose: bool) -> ProgressLayout { + ProgressLayout { + total: 3, + name_width: 24, + columns: 80, + verbose, + } +} + +pub(super) fn model(name: &str, status: &str) -> ModelResultText { + ModelResultText { + name: name.to_owned(), + status: status.to_owned(), + duration_ms: Some(1234.0), + ..ModelResultText::default() + } +} + +pub(super) fn failed_model(name: &str, message: &str) -> ModelResultText { + ModelResultText { + failed_phase: Some("staging".to_owned()), + error: ErrorText { + code: Some("E100".to_owned()), + message: Some(message.to_owned()), + help: None, + }, + ..model(name, "failed") + } +} + +pub(super) fn audit(name: &str, outcome: &str, row_count: i64) -> AuditResultText { + AuditResultText { + audit_name: name.to_owned(), + run_scope_phase: "final".to_owned(), + outcome: outcome.to_owned(), + row_count, + measured_value: "None".to_owned(), + ..AuditResultText::default() + } +} + +pub(super) fn test(label: &str, outcome: &str, step_outcome: &str) -> TestResultText { + TestResultText { + label: label.to_owned(), + outcome: outcome.to_owned(), + steps: vec![TestStepText { + model_name: "orders".to_owned(), + outcome: step_outcome.to_owned(), + detail: String::new(), + }], + error: ErrorText::default(), + } +} diff --git a/crates/sqlbuild-core/src/build_progress/tests/test_build_progress.rs b/crates/sqlbuild-core/src/build_progress/tests/test_build_progress.rs new file mode 100644 index 0000000000..1fac3f177c --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/tests/test_build_progress.rs @@ -0,0 +1,295 @@ +use crate::build_progress::main::format_build_footer::format_build_footer; +use crate::build_progress::main::progress_name_width::progress_name_width; +use crate::build_progress::main::render_build_spinner_line::render_build_spinner_line; +use crate::build_progress::main::render_failed_test_result::render_failed_test_result; +use crate::build_progress::main::render_model_result::render_model_result; +use crate::build_progress::main::scheduler_diagnostic_message::scheduler_diagnostic_message; +use crate::build_progress::models::{ + BuildResultText, ErrorText, LifecycleEventText, ModelResultText, ModelRowFacts, + NameWidthInputs, ProgressLayout, PythonNodeText, SchedulerStateText, SpinnerRow, +}; +use crate::build_progress::tests::helpers::{audit, failed_model, layout, model, test}; +use crate::build_progress::tests::test_types::{ + FailedTestTestCase, FooterTestCase, InlineErrorTestCase, ModelRowTestCase, NameWidthTestCase, + SchedulerTestCase, SpinnerTestCase, +}; +use crate::cli_text::main::format_inline_error_lines::format_inline_error_lines; +use crate::cli_text::models::CliStyle; + +const PLAIN: CliStyle = CliStyle { use_color: false }; + +#[test] +fn given_build_results_when_formatting_footer_then_matches_python_layout() { + let test_cases = [ + FooterTestCase { + description: "a clean run reports success", + result: BuildResultText { + models: vec![model("orders", "success")], + ..BuildResultText::default() + }, + python: Vec::new(), + use_color: false, + expected_text: "\u{2713} Completed successfully PASS=1 WARN=0 FAIL=0 INSUFFICIENT=0 SKIP=0 TOTAL=1 (1.50s)", + }, + FooterTestCase { + description: "warnings, failures and skipped nodes each get a section", + result: BuildResultText { + failed: true, + models: vec![ + ModelResultText { + audits: vec![audit("not_null", "warn", 2)], + ..model("orders", "success") + }, + ModelResultText { + staging_relation: Some("orders__delta".to_owned()), + ..failed_model("customers", "boom") + }, + model("payments", "skipped"), + ], + ..BuildResultText::default() + }, + python: vec![PythonNodeText { + name: "refresh".to_owned(), + kind: "task".to_owned(), + status: "failed".to_owned(), + error_message: None, + warnings: Vec::new(), + }], + use_color: false, + expected_text: "\nWarnings:\n\n orders\n audit not_null returned 2 rows\n\n\u{2717} Completed with errors PASS=1 WARN=1 FAIL=2 INSUFFICIENT=0 SKIP=1 TOTAL=5 (1.50s)\n\nFailures:\n\n customers (staging)\n error [E100] boom\n delta table kept for inspection: orders__delta\n\n\nPython node failures:\n\n task refresh Python node failed\n\nSkipped (1): payments", + }, + ]; + + for test_case in test_cases { + let text: String = format_build_footer( + &test_case.result, + &test_case.python, + 1.5, + CliStyle { + use_color: test_case.use_color, + }, + ); + assert_eq!(text, test_case.expected_text, "{}", test_case.description); + } +} + +#[test] +fn given_model_result_when_rendering_then_nests_tests_audits_and_errors() { + let test_cases = [ + ModelRowTestCase { + description: "a passing model lists its tests and aggregated audits", + model: ModelResultText { + audits: vec![audit("unique", "error", 1), audit("unique", "pass", 0)], + ..model("orders", "success") + }, + tests: vec![test("orders_test", "pass", "pass")], + verbose: false, + expected_text: " 2/3 table orders OK 1.23s\n test orders_test PASS\n expect expected orders PASS\n audit unique FAIL 1/2 1 row\n", + }, + ModelRowTestCase { + description: "a failed model shows its phase and wrapped coded error", + model: failed_model("customers", "boom"), + tests: Vec::new(), + verbose: false, + expected_text: " 2/3 table customers FAIL 1.23s staging\n error [E100] boom\n", + }, + ModelRowTestCase { + description: "verbose output prints lifecycle SQL with a semicolon", + model: ModelResultText { + events: vec![LifecycleEventText::Sql("select 1 ".to_owned())], + batch_count: Some(2), + rows_affected: Some(12_345), + ..model("orders", "success") + }, + tests: Vec::new(), + verbose: true, + expected_text: " 2/3 table orders OK 1.23s\n\n select 1;\n\n 2 batches 12.3K rows\n", + }, + ]; + + for test_case in test_cases { + let facts: ModelRowFacts = ModelRowFacts { + display_type: "table".to_owned(), + ..ModelRowFacts::default() + }; + assert_eq!( + render_model_result( + (&test_case.model, &facts, &test_case.tests), + 2, + layout(test_case.verbose), + PLAIN + ), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_failed_test_when_rendering_then_draws_tree_children() { + let test_cases = [ + FailedTestTestCase { + description: "the last expectation closes the tree", + test: test("orders_test", "fail", "fail"), + expected_text: " test orders_test FAIL\n \u{2514}\u{2500}\u{2500} expect expected orders FAIL\n", + }, + FailedTestTestCase { + description: "an error closes the tree after the expectations", + test: crate::build_progress::models::TestResultText { + error: ErrorText { + code: None, + message: Some("broken".to_owned()), + help: None, + }, + ..test("orders_test", "error", "error") + }, + expected_text: " test orders_test ERROR\n \u{251c}\u{2500}\u{2500} expect expected orders ERROR\n \u{2514}\u{2500}\u{2500} error broken\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_failed_test_result(&test_case.test, layout(false), PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_running_node_when_drawing_spinner_then_fits_terminal() { + let test_cases = [ + SpinnerTestCase { + description: "a wide terminal shows the full row", + row: SpinnerRow { + position: 1, + resource_kind: "table".to_owned(), + name: "orders".to_owned(), + message: "batch 1/2".to_owned(), + }, + columns: 120, + expected_text: "\r\x1b[K 1/3 table orders batch 1/2 \u{280b}", + }, + SpinnerTestCase { + description: "a narrow terminal truncates the row", + row: SpinnerRow { + position: 1, + resource_kind: "table".to_owned(), + name: "orders".to_owned(), + message: String::new(), + }, + columns: 24, + expected_text: "\r\x1b[K 1/3 table o... \u{280b}", + }, + ]; + + for test_case in test_cases { + let layout: ProgressLayout = ProgressLayout { + columns: test_case.columns, + ..layout(false) + }; + assert_eq!( + render_build_spinner_line(&test_case.row, 0, layout, PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_row_names_when_sizing_then_tree_children_keep_status_column() { + let test_cases = [ + NameWidthTestCase { + description: "short names use the minimum width", + inputs: NameWidthInputs { + names: vec!["orders".to_owned()], + tree_child_names: Vec::new(), + }, + expected_width: 20, + }, + NameWidthTestCase { + description: "expectation children add their tree indent", + inputs: NameWidthInputs { + names: vec!["orders".to_owned()], + tree_child_names: vec!["a_rather_long_model_name".to_owned()], + }, + expected_width: 39, + }, + ]; + + for test_case in test_cases { + assert_eq!( + progress_name_width(&test_case.inputs), + test_case.expected_width, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_scheduler_state_when_describing_then_flags_frontier_constraints() { + let test_cases = [ + SchedulerTestCase { + description: "an idle worker with only waiting nodes is frontier constrained", + state: SchedulerStateText { + running: 1, + ready: 0, + waiting: 3, + limit: 4, + aborted: 0, + }, + expected_text: "Scheduler 1 running, 0 ready, 3 waiting on dependencies, limit 4 (DAG frontier constrained)", + }, + SchedulerTestCase { + description: "aborted nodes are reported after a stop", + state: SchedulerStateText { + running: 0, + ready: 1, + waiting: 2, + limit: 4, + aborted: 5, + }, + expected_text: "Scheduler 0 running, 1 ready, 2 waiting on dependencies, 5 aborted after stop, limit 4", + }, + ]; + + for test_case in test_cases { + assert_eq!( + scheduler_diagnostic_message(test_case.state), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_long_error_when_wrapping_inline_then_truncates_to_two_lines() { + let test_cases = [ + InlineErrorTestCase { + description: "a coded error with help wraps and keeps the code first", + error: (Some("E1"), "short", Some("do this")), + width: 40, + expected_lines: &["[E1] short", "= help: do this"], + }, + InlineErrorTestCase { + description: "hyphenated words are not split and overflow is truncated", + error: (None, "alpha-beta-gamma delta epsilon zeta eta", None), + width: 12, + expected_lines: &["alpha-beta-g", "amma delt..."], + }, + ]; + + for test_case in test_cases { + assert_eq!( + format_inline_error_lines(test_case.error, test_case.width, PLAIN), + test_case.expected_lines, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/build_progress/tests/test_types.rs b/crates/sqlbuild-core/src/build_progress/tests/test_types.rs new file mode 100644 index 0000000000..a258d7dd49 --- /dev/null +++ b/crates/sqlbuild-core/src/build_progress/tests/test_types.rs @@ -0,0 +1,52 @@ +use crate::build_progress::models::{ + BuildResultText, ModelResultText, NameWidthInputs, PythonNodeText, SchedulerStateText, + SpinnerRow, TestResultText, +}; + +pub(super) struct FooterTestCase { + pub(super) description: &'static str, + pub(super) result: BuildResultText, + pub(super) python: Vec, + pub(super) use_color: bool, + pub(super) expected_text: &'static str, +} + +pub(super) struct ModelRowTestCase { + pub(super) description: &'static str, + pub(super) model: ModelResultText, + pub(super) tests: Vec, + pub(super) verbose: bool, + pub(super) expected_text: &'static str, +} + +pub(super) struct FailedTestTestCase { + pub(super) description: &'static str, + pub(super) test: TestResultText, + pub(super) expected_text: &'static str, +} + +pub(super) struct SpinnerTestCase { + pub(super) description: &'static str, + pub(super) row: SpinnerRow, + pub(super) columns: usize, + pub(super) expected_text: &'static str, +} + +pub(super) struct NameWidthTestCase { + pub(super) description: &'static str, + pub(super) inputs: NameWidthInputs, + pub(super) expected_width: usize, +} + +pub(super) struct SchedulerTestCase { + pub(super) description: &'static str, + pub(super) state: SchedulerStateText, + pub(super) expected_text: &'static str, +} + +pub(super) struct InlineErrorTestCase { + pub(super) description: &'static str, + pub(super) error: (Option<&'static str>, &'static str, Option<&'static str>), + pub(super) width: usize, + pub(super) expected_lines: &'static [&'static str], +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/coded_error.rs b/crates/sqlbuild-core/src/cli_text/_helpers/coded_error.rs index b80362ec49..fc0695668a 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/coded_error.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/coded_error.rs @@ -1,6 +1,10 @@ -//! A result-stored coded error, as CLI expected errors render it. +//! Result-stored coded errors, as CLI expected errors and compact inline rows render them. +use crate::cli_text::constants::MAX_INLINE_ERROR_LINES; use crate::cli_text::models::CliStyle; +use crate::text::main::python_isspace::python_isspace; +use crate::text::main::python_splitlines::python_splitlines; +use crate::text::main::text_wrap::text_wrap; /// `error[CODE]: message` or `[CODE] message`, later lines muted, then any help line. pub(crate) fn coded_error( @@ -29,3 +33,63 @@ pub(crate) fn coded_error( } rendered } + +/// `[code] message`, with `= help: help` on its own paragraph. +pub(crate) fn inline_error_text(code_prefix: &str, message: &str, help: Option<&str>) -> String { + let text: String = if code_prefix.is_empty() { + message.to_owned() + } else { + format!("{code_prefix} {message}") + }; + match help { + Some(help) => format!("{text}\n= help: {help}"), + None => text, + } +} + +/// Every paragraph wrapped on whitespace only, an empty paragraph kept as one empty line. +pub(crate) fn wrap_paragraphs(text: &str, width: usize) -> Vec { + let mut paragraphs: Vec<&str> = python_splitlines(text); + if paragraphs.is_empty() { + paragraphs.push(text); + } + let mut wrapped: Vec = Vec::new(); + for paragraph in paragraphs { + let lines: Vec = text_wrap(paragraph, width, ("", ""), false); + if lines.is_empty() { + wrapped.push(String::new()); + } else { + wrapped.extend(lines); + } + } + wrapped +} + +/// The first lines, the last one cut to end in an ellipsis when lines were dropped. +pub(crate) fn truncate_lines(mut lines: Vec, width: usize) -> Vec { + let truncated: bool = lines.len() > MAX_INLINE_ERROR_LINES; + lines.truncate(MAX_INLINE_ERROR_LINES); + if truncated && let Some(last) = lines.last_mut() { + let ellipsis: String = ".".repeat(width.min(3)); + let kept: String = last + .chars() + .take(width.saturating_sub(ellipsis.len())) + .collect(); + *last = format!("{}{ellipsis}", kept.trim_end_matches(python_isspace)); + } + lines +} + +/// The first occurrence of the code prefix in the first line, styled as strong error text. +pub(crate) fn style_code_prefix( + mut lines: Vec, + code_prefix: &str, + style: CliStyle, +) -> Vec { + if !code_prefix.is_empty() + && let Some(first) = lines.first_mut() + { + *first = first.replacen(code_prefix, &style.error_strong(code_prefix), 1); + } + lines +} diff --git a/crates/sqlbuild-core/src/cli_text/constants.rs b/crates/sqlbuild-core/src/cli_text/constants.rs index 6e575a762f..e84eb2526f 100644 --- a/crates/sqlbuild-core/src/cli_text/constants.rs +++ b/crates/sqlbuild-core/src/cli_text/constants.rs @@ -87,3 +87,5 @@ pub(crate) const COUNT_HEADER_OPENING: &str = " ("; pub(crate) const COUNT_HEADER_CLOSING: &str = ")"; pub(crate) const DBT_NO_MATCH_WARNING: &str = "No dbt or SQLBuild resources matched the selection."; pub(crate) const DBT_RESOURCE_FALLBACK_TYPE: &str = "resource"; +/// An inline error wraps to at most this many lines before it is truncated. +pub(crate) const MAX_INLINE_ERROR_LINES: usize = 2; diff --git a/crates/sqlbuild-core/src/cli_text/main/format_inline_error_lines.rs b/crates/sqlbuild-core/src/cli_text/main/format_inline_error_lines.rs new file mode 100644 index 0000000000..f8d9878a04 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_inline_error_lines.rs @@ -0,0 +1,21 @@ +//! A compact inline diagnostic wrapped to two display lines. + +use crate::cli_text::_helpers::coded_error::{ + inline_error_text, style_code_prefix, truncate_lines, wrap_paragraphs, +}; +use crate::cli_text::models::CliStyle; + +/// The `[code] message` and `= help:` text wrapped to two lines, truncated with an ellipsis. +#[must_use] +pub fn format_inline_error_lines( + (code, message, help): (Option<&str>, &str, Option<&str>), + content_width: usize, + style: CliStyle, +) -> Vec { + let code_prefix: String = code.map_or_else(String::new, |code| format!("[{code}]")); + let wrapped: Vec = wrap_paragraphs( + &inline_error_text(&code_prefix, message, help), + content_width, + ); + style_code_prefix(truncate_lines(wrapped, content_width), &code_prefix, style) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/mod.rs b/crates/sqlbuild-core/src/cli_text/main/mod.rs index 8208efb9bd..60ec06a502 100644 --- a/crates/sqlbuild-core/src/cli_text/main/mod.rs +++ b/crates/sqlbuild-core/src/cli_text/main/mod.rs @@ -6,6 +6,7 @@ pub mod format_completion_line; pub mod format_count_noun; pub mod format_dbt_interop_plan; pub mod format_dbt_interop_plan_json; +pub mod format_inline_error_lines; pub mod format_overflow_line; pub mod format_phase_line; pub mod format_plan_completion; diff --git a/crates/sqlbuild-core/src/format_command/_helpers/diagnostics.rs b/crates/sqlbuild-core/src/format_command/_helpers/diagnostics.rs index 5a98a55555..aed12e2d2f 100644 --- a/crates/sqlbuild-core/src/format_command/_helpers/diagnostics.rs +++ b/crates/sqlbuild-core/src/format_command/_helpers/diagnostics.rs @@ -67,6 +67,7 @@ fn help_lines(remediation: &str, style: CliStyle) -> Vec { remediation, REPORT_LINE_WIDTH, (HELP_PREFIX, HELP_CONTINUATION), + true, ); let Some((first, rest)) = wrapped.split_first() else { return Vec::new(); diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index e143b3605c..a771914e28 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -4,6 +4,7 @@ pub mod audit_command; pub mod build_command; +pub mod build_progress; pub mod check_command; pub mod cli_text; pub mod clone_text; diff --git a/crates/sqlbuild-core/src/text/_helpers/text_wrap.rs b/crates/sqlbuild-core/src/text/_helpers/text_wrap.rs index 28ed46aa88..e9ac1b13d5 100644 --- a/crates/sqlbuild-core/src/text/_helpers/text_wrap.rs +++ b/crates/sqlbuild-core/src/text/_helpers/text_wrap.rs @@ -32,10 +32,19 @@ fn munge_whitespace(text: &str) -> Vec { munged } -/// `textwrap.wrap(text, width, initial_indent=..., subsequent_indent=...)`. -pub(crate) fn wrap(text: &str, width: usize, (initial, subsequent): (&str, &str)) -> Vec { +/// `textwrap.wrap(text, width, initial_indent=..., subsequent_indent=..., break_on_hyphens=...)`. +pub(crate) fn wrap( + text: &str, + width: usize, + (initial, subsequent): (&str, &str), + break_on_hyphens: bool, +) -> Vec { let characters: Vec = munge_whitespace(text); - let mut chunks: Vec> = split_chunks(&characters); + let mut chunks: Vec> = if break_on_hyphens { + split_chunks(&characters) + } else { + split_whitespace_chunks(&characters) + }; chunks.reverse(); let mut lines: Vec = Vec::new(); while !chunks.is_empty() { @@ -59,7 +68,12 @@ pub(crate) fn wrap(text: &str, width: usize, (initial, subsequent): (&str, &str) } } if let Some(chunk) = chunks.pop_if(|chunk| chunk.len() as i64 > available) { - let (head, rest) = split_long_word(chunk, line.is_empty(), length, available); + let (head, rest) = split_long_word( + chunk, + (line.is_empty(), break_on_hyphens), + length, + available, + ); line.extend(head); chunks.extend(rest); } @@ -77,7 +91,7 @@ pub(crate) fn wrap(text: &str, width: usize, (initial, subsequent): (&str, &str) /// `_handle_long_word`: the part of a too-long chunk the line takes, and what stays queued. fn split_long_word( mut chunk: Vec, - line_is_empty: bool, + (line_is_empty, break_on_hyphens): (bool, bool), length: usize, width: i64, ) -> (Option>, Option>) { @@ -91,7 +105,8 @@ fn split_long_word( } let space_left: usize = space_left as usize; let mut end: usize = space_left.min(chunk.len()); - if chunk.len() > space_left + if break_on_hyphens + && chunk.len() > space_left && let Some(hyphen) = chunk[..space_left] .iter() .rposition(|character| *character == '-') @@ -136,6 +151,22 @@ fn split_chunks(text: &[char]) -> Vec> { chunks } +/// `TextWrapper.wordsep_simple_re.split`: whitespace runs and the words between them. +fn split_whitespace_chunks(text: &[char]) -> Vec> { + let mut chunks: Vec> = Vec::new(); + let mut start: usize = 0; + while start < text.len() { + let space: bool = is_space(text[start]); + let length: usize = text[start..] + .iter() + .take_while(|character| is_space(**character) == space) + .count(); + chunks.push(text[start..start + length].to_vec()); + start += length; + } + chunks +} + fn at(text: &[char], index: usize) -> Option { text.get(index).copied() } diff --git a/crates/sqlbuild-core/src/text/main/text_wrap.rs b/crates/sqlbuild-core/src/text/main/text_wrap.rs index b2e32cccee..9f72651762 100644 --- a/crates/sqlbuild-core/src/text/main/text_wrap.rs +++ b/crates/sqlbuild-core/src/text/main/text_wrap.rs @@ -1,9 +1,14 @@ -//! Python's `textwrap.wrap` with its default options. +//! Python's `textwrap.wrap`. use crate::text::_helpers::text_wrap::wrap; -/// `textwrap.wrap(text, width, initial_indent=initial, subsequent_indent=subsequent)`. +/// `textwrap.wrap(text, width, initial_indent=initial, subsequent_indent=subsequent, break_on_hyphens=...)`. #[must_use] -pub fn text_wrap(text: &str, width: usize, indents: (&str, &str)) -> Vec { - wrap(text, width, indents) +pub fn text_wrap( + text: &str, + width: usize, + indents: (&str, &str), + break_on_hyphens: bool, +) -> Vec { + wrap(text, width, indents, break_on_hyphens) } diff --git a/crates/sqlbuild-core/src/text/tests/test_text_wrap.rs b/crates/sqlbuild-core/src/text/tests/test_text_wrap.rs index 7aabeeb99a..5126658f43 100644 --- a/crates/sqlbuild-core/src/text/tests/test_text_wrap.rs +++ b/crates/sqlbuild-core/src/text/tests/test_text_wrap.rs @@ -33,7 +33,7 @@ fn given_prose_when_wrapping_then_lines_match_python_textwrap() { for test_case in test_cases { assert_eq!( - text_wrap(test_case.text, test_case.width, test_case.indents), + text_wrap(test_case.text, test_case.width, test_case.indents, true), test_case.expected_lines, "{}", test_case.description diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs new file mode 100644 index 0000000000..15b9afca2f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs @@ -0,0 +1,36 @@ +//! The `sqb build` completion footer. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::build_progress::main::format_build_footer::format_build_footer as footer; +use sqlbuild_core::build_progress::models::PythonNodeText; +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::bindings::_helpers::build_progress::result_text::{build_text, python_node_text}; + +/// Render the build completion line, warnings, failures and skipped rollup. +#[pyfunction] +#[pyo3(signature = (*, result, elapsed, use_color, python_node_results = None))] +fn format_build_footer( + result: &Bound<'_, PyAny>, + elapsed: f64, + use_color: bool, + python_node_results: Option<&Bound<'_, PyAny>>, +) -> PyResult { + let mut python: Vec = Vec::new(); + if let Some(results) = python_node_results { + for node in results.try_iter()? { + python.push(python_node_text(&node?)?); + } + } + Ok(footer( + &build_text(result)?, + &python, + elapsed, + CliStyle { use_color }, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_build_footer, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs new file mode 100644 index 0000000000..2090874a94 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod footer; +pub(crate) mod registration; +pub(crate) mod result_text; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs new file mode 100644 index 0000000000..52c90493f2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs @@ -0,0 +1,9 @@ +//! Register the native build progress reporter and footer. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::build_progress::footer; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + footer::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs new file mode 100644 index 0000000000..d02c1b6bad --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs @@ -0,0 +1,243 @@ +//! Executor result objects read into the plain data build progress output renders. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_core::build_progress::models::{ + AuditResultText, BuildResultText, ErrorText, HookResultText, LifecycleEventText, LoadRowText, + ModelResultText, NodeResultText, PythonNodeText, SkipText, TestResultText, TestStepText, +}; +use sqlbuild_core::progress::main::format_expectation_detail::format_expectation_detail; + +use crate::bindings::_helpers::cli::progress_text::expectation_step; +use crate::bindings::_helpers::plan_output::display::{cursor_bound_display, inclusive_cursor_end}; +use crate::bindings::_helpers::sql_test_command::test_command::test_label; + +const SQL_EVENT: &str = "sql"; +const PRE_HOOKS: &str = "pre_hooks"; +const MEASUREMENT: &str = "measurement"; +const FAILED: &str = "failed"; + +pub(crate) fn text(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + Ok(value.getattr(name)?.str()?.to_string()) +} + +pub(crate) fn optional(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + let field: Bound<'_, PyAny> = value.getattr(name)?; + if field.is_none() { + return Ok(None); + } + Ok(Some(field.str()?.to_string())) +} + +fn texts(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + value + .getattr(name)? + .try_iter()? + .map(|item| Ok(item?.str()?.to_string())) + .collect() +} + +fn items<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult>> { + value.getattr(name)?.try_iter()?.collect() +} + +fn duration(value: &Bound<'_, PyAny>) -> PyResult> { + value.getattr("duration_ms")?.extract() +} + +fn error(value: &Bound<'_, PyAny>) -> PyResult { + Ok(ErrorText { + code: optional(value, "error_code")?, + message: optional(value, "error_message")?, + help: optional(value, "error_help")?, + }) +} + +fn skip(value: &Bound<'_, PyAny>) -> PyResult { + Ok(SkipText { + mode: optional(value, "skip_mode")?, + reason: optional(value, "skip_reason")?, + }) +} + +fn events(value: &Bound<'_, PyAny>) -> PyResult> { + let mut events: Vec = Vec::new(); + for event in items(value, "lifecycle_events")? { + let content: String = event.getattr("content")?.extract()?; + events.push(if text(&event, "kind")? == SQL_EVENT { + LifecycleEventText::Sql(content) + } else { + LifecycleEventText::Log(content) + }); + } + Ok(events) +} + +pub(crate) fn audit_text(audit: &Bound<'_, PyAny>) -> PyResult { + Ok(AuditResultText { + audit_name: text(audit, "audit_name")?, + attached_column_name: optional(audit, "attached_column_name")?, + attached_target_name: optional(audit, "attached_target_name")?, + run_scope_phase: text(audit, "run_scope_phase")?, + outcome: text(audit, "outcome")?, + row_count: audit.getattr("row_count")?.extract()?, + reused: audit.getattr("reused")?.is_truthy()?, + executed_sql: optional(audit, "executed_sql")?, + measurement: text(audit, "evaluation_mode")? == MEASUREMENT, + measured_value: text(audit, "measured_value")?, + execution_error: optional(audit, "execution_error")?, + }) +} + +fn audits(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + items(value, name)?.iter().map(audit_text).collect() +} + +fn hook_text(hook: &Bound<'_, PyAny>) -> PyResult { + Ok(HookResultText { + pre_hook: text(hook, "phase")? == PRE_HOOKS, + hook_type: text(hook, "hook_type")?, + label: text(hook, "label")?, + status: text(hook, "status")?, + skip: skip(hook)?, + }) +} + +fn cursor_range(model: &Bound<'_, PyAny>) -> PyResult> { + let start: Bound<'_, PyAny> = model.getattr("cursor_range_start")?; + let end: Bound<'_, PyAny> = model.getattr("cursor_range_end")?; + if start.is_none() || end.is_none() || model.getattr("empty_cursor_inputs")?.is_truthy()? { + return Ok(None); + } + let cursor_type: Bound<'_, PyAny> = model.getattr("cursor_type")?; + let cursor_grain: Bound<'_, PyAny> = model.getattr("cursor_grain")?; + Ok(Some(( + cursor_bound_display(&start, &cursor_type, &cursor_grain)?, + inclusive_cursor_end(&end, &cursor_type, &cursor_grain)?, + ))) +} + +pub(crate) fn model_text(model: &Bound<'_, PyAny>) -> PyResult { + Ok(ModelResultText { + name: text(model, "model_name")?, + status: text(model, "status")?, + failed_phase: optional(model, "failed_phase")?, + staging_relation: optional(model, "staging_relation")?, + duration_ms: duration(model)?, + batch_count: model.getattr("batch_count")?.extract()?, + batch_size: optional(model, "batch_size")?, + rows_affected: model.getattr("rows_affected")?.extract()?, + cursor_range: cursor_range(model)?, + empty_cursor_inputs: texts(model, "empty_cursor_inputs")?, + waiting_on_empty_inputs: model.getattr("waiting_on_empty_inputs")?.is_truthy()?, + recovery_batch_count: model + .getattr("microbatch_recovery_batch_count")? + .extract()?, + synthetic_completion_count: model + .getattr("microbatch_synthetic_completion_count")? + .extract()?, + replay_requirement_state: optional(model, "microbatch_replay_requirement_state")?, + audits: audits(model, "audit_results")?, + warnings: texts(model, "warning_messages")?, + events: events(model)?, + hooks: items(model, "hook_results")? + .iter() + .map(hook_text) + .collect::>()?, + skip: skip(model)?, + error: error(model)?, + }) +} + +pub(crate) fn seed_text(seed: &Bound<'_, PyAny>) -> PyResult { + Ok(NodeResultText { + name: text(seed, "seed_name")?, + kind: "seed".to_owned(), + status: text(seed, "status")?, + duration_ms: duration(seed)?, + error: error(seed)?, + warnings: Vec::new(), + }) +} + +pub(crate) fn function_text(function: &Bound<'_, PyAny>) -> PyResult { + Ok(NodeResultText { + name: text(function, "function_name")?, + kind: text(function, "function_kind")?, + status: text(function, "status")?, + duration_ms: duration(function)?, + error: error(function)?, + warnings: texts(function, "warning_messages")?, + }) +} + +pub(crate) fn load_text(load: &Bound<'_, PyAny>) -> PyResult { + Ok(LoadRowText { + source_name: text(load, "source_name")?, + loader_name: text(load, "loader_name")?, + resource_kind: text(load, "resource_kind")?, + status: text(load, "status")?, + rows_loaded: load.getattr("rows_loaded")?.extract()?, + duration_ms: duration(load)?, + events: events(load)?, + skip: skip(load)?, + error_message: optional(load, "error_message")?, + warnings: texts(load, "warning_messages")?, + }) +} + +pub(crate) fn test_text(test: &Bound<'_, PyAny>) -> PyResult { + let mut steps: Vec = Vec::new(); + for step in items(test, "step_results")? { + steps.push(TestStepText { + model_name: text(&step, "model_name")?, + outcome: text(&step, "outcome")?, + detail: format_expectation_detail(&expectation_step(&step)?), + }); + } + Ok(TestResultText { + label: test_label(test, "test_name")?, + outcome: text(test, "outcome")?, + steps, + error: error(test)?, + }) +} + +pub(crate) fn python_node_text(node: &Bound<'_, PyAny>) -> PyResult { + Ok(PythonNodeText { + name: text(node, "node_name")?, + kind: text(node, "kind")?, + status: text(node, "status")?, + error_message: optional(node, "error_message")?, + warnings: texts(node, "warning_messages")?, + }) +} + +pub(crate) fn build_text(result: &Bound<'_, PyAny>) -> PyResult { + Ok(BuildResultText { + failed: text(result, "status")? == FAILED, + warning_count: result.getattr("warning_count")?.extract()?, + insufficient_count: result.getattr("insufficient_count")?.extract()?, + models: items(result, "model_results")? + .iter() + .map(model_text) + .collect::>()?, + seeds: items(result, "seed_results")? + .iter() + .map(seed_text) + .collect::>()?, + functions: items(result, "function_results")? + .iter() + .map(function_text) + .collect::>()?, + loads: items(result, "load_results")? + .iter() + .map(load_text) + .collect::>()?, + tests: items(result, "test_results")? + .iter() + .map(test_text) + .collect::>()?, + source_audits: audits(result, "source_audit_results")?, + end_audits: audits(result, "end_audit_results")?, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs index f5e80e4a5f..1df6d8f56d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs @@ -63,7 +63,7 @@ fn difference_samples(samples: &Bound<'_, PyAny>) -> PyResult) -> PyResult { +pub(crate) fn expectation_step(step: &Bound<'_, PyAny>) -> PyResult { let mut column_differences: Vec<(String, String, String)> = Vec::new(); for difference in step.getattr("column_differences")?.try_iter()? { let difference: Bound<'_, PyAny> = difference?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 58be3c4671..7e12aa65a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -4,10 +4,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::cli::{argv, entry, progress, progress_reporters, progress_text}; use crate::bindings::_helpers::{ - audit_command, build_command, check, clone, command_runtime, contract_command, cost_command, - cost_report, dbt_command, dbt_events, dbt_interop, debug_command, diff_report, format_command, - freshness, init_command, janitor, load, presentation_text, query_command, result_output, - scenario_command, seed_command, sql_test_command, workspace_commands, + audit_command, build_command, build_progress, check, clone, command_runtime, contract_command, + cost_command, cost_report, dbt_command, dbt_events, dbt_interop, debug_command, diff_report, + format_command, freshness, init_command, janitor, load, presentation_text, query_command, + result_output, scenario_command, seed_command, sql_test_command, workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -34,6 +34,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { seed_command::registration::register(module)?; audit_command::registration::register(module)?; build_command::registration::register(module)?; + build_progress::registration::register(module)?; query_command::registration::register(module)?; scenario_command::registration::register(module)?; contract_command::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 8b309cf090..e5a1a65ec8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod auditing; pub(crate) mod authored_values; pub(crate) mod boundary; pub(crate) mod build_command; +pub(crate) mod build_progress; pub(crate) mod cache; pub(crate) mod check; pub(crate) mod cli; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs index 52abfd9a36..c5da29096c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs @@ -44,7 +44,7 @@ fn label_facts(entry: &Bound<'_, PyAny>) -> PyResult { }) } -fn optional_facts(entry: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn optional_facts(entry: &Bound<'_, PyAny>) -> PyResult> { if entry.is_none() { return Ok(None); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 1a0a4d7d90..beb0610944 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -79,7 +79,7 @@ fn compact_json(payload: &Bound<'_, PyAny>) -> PyResult { } /// The label of one test or result: its name, plus source and values for parameterized cases. -fn test_label(item: &Bound<'_, PyAny>, name_field: &str) -> PyResult { +pub(crate) fn test_label(item: &Bound<'_, PyAny>, name_field: &str) -> PyResult { let name: String = item.getattr(name_field)?.extract()?; let source_path: Bound<'_, PyAny> = item.getattr("source_path")?; let values: Vec> = item diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index bb9aee70a3..70863c345a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1744,6 +1744,13 @@ def write_execution_header( ) -> None: ... def format_expectation_name(model_name: str) -> str: ... def format_expectation_detail(step_result: StepResult) -> str: ... +def format_build_footer( + *, + result: BuildExecutionResult, + elapsed: float, + use_color: bool, + python_node_results: tuple[PythonNodeExecutionResult, ...] | None = None, +) -> str: ... def format_build_execution_json( *, result: BuildExecutionResult, From 7aeada1d58b0b2aa6940e8d6add55b0ab14299cf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 03:27:23 +0100 Subject: [PATCH 02/40] refactor: port the audit and test progress reporters to native classes --- crates/sqlbuild-core/src/lib.rs | 1 + .../src/nested_progress/constants.rs | 13 + .../nested_progress/main/audit_group_name.rs | 13 + .../nested_progress/main/audit_resource_id.rs | 17 + .../src/nested_progress/main/mod.rs | 7 + .../main/render_audit_aggregate.rs | 13 + .../nested_progress/main/render_audit_row.rs | 39 ++ .../main/render_group_header.rs | 10 + .../main/render_nested_item.rs | 59 +++ .../main/render_nested_spinner_line.rs | 36 ++ .../sqlbuild-core/src/nested_progress/mod.rs | 8 + .../src/nested_progress/models.rs | 39 ++ .../src/nested_progress/tests.rs | 6 + .../src/nested_progress/tests/helpers.rs | 9 + .../tests/test_nested_progress.rs | 87 ++++ .../src/nested_progress/tests/test_types.rs | 14 + .../_helpers/audit_command/command.rs | 3 +- .../_helpers/build_progress/audit_progress.rs | 304 ++++++++++++++ .../bindings/_helpers/build_progress/mod.rs | 3 + .../build_progress/nested_progress.rs | 392 ++++++++++++++++++ .../_helpers/build_progress/registration.rs | 8 +- .../_helpers/build_progress/terminal.rs | 118 ++++++ .../_helpers/sql_test_command/test_command.rs | 4 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 49 +++ src/sqlbuild/cli/progress/classes/__init__.py | 1 - .../classes/audit_progress_reporter.py | 198 --------- .../nested_command_progress_callbacks.py | 288 ------------- src/sqlbuild/cli/progress/constants.py | 3 - .../commands/shared/_helpers/test_progress.py | 4 +- .../pipeline/_helpers/test_auditing.py | 2 +- 31 files changed, 1247 insertions(+), 503 deletions(-) create mode 100644 crates/sqlbuild-core/src/nested_progress/constants.rs create mode 100644 crates/sqlbuild-core/src/nested_progress/main/audit_group_name.rs create mode 100644 crates/sqlbuild-core/src/nested_progress/main/audit_resource_id.rs create mode 100644 crates/sqlbuild-core/src/nested_progress/main/mod.rs create mode 100644 crates/sqlbuild-core/src/nested_progress/main/render_audit_aggregate.rs create mode 100644 crates/sqlbuild-core/src/nested_progress/main/render_audit_row.rs create mode 100644 crates/sqlbuild-core/src/nested_progress/main/render_group_header.rs create mode 100644 crates/sqlbuild-core/src/nested_progress/main/render_nested_item.rs create mode 100644 crates/sqlbuild-core/src/nested_progress/main/render_nested_spinner_line.rs create mode 100644 crates/sqlbuild-core/src/nested_progress/mod.rs create mode 100644 crates/sqlbuild-core/src/nested_progress/models.rs create mode 100644 crates/sqlbuild-core/src/nested_progress/tests.rs create mode 100644 crates/sqlbuild-core/src/nested_progress/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/nested_progress/tests/test_nested_progress.rs create mode 100644 crates/sqlbuild-core/src/nested_progress/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs delete mode 100644 src/sqlbuild/cli/progress/classes/__init__.py delete mode 100644 src/sqlbuild/cli/progress/classes/audit_progress_reporter.py delete mode 100644 src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py delete mode 100644 src/sqlbuild/cli/progress/constants.py diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index a771914e28..a363d58d5c 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -26,6 +26,7 @@ pub mod janitor_text; pub mod json; pub mod load_command; pub mod models; +pub mod nested_progress; pub mod panics; pub mod plan_display; pub mod posix_path; diff --git a/crates/sqlbuild-core/src/nested_progress/constants.rs b/crates/sqlbuild-core/src/nested_progress/constants.rs new file mode 100644 index 0000000000..bdddc490b7 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/constants.rs @@ -0,0 +1,13 @@ +//! Column widths of grouped command progress rows. + +pub(crate) const LABEL_WIDTH: usize = 10; +pub(crate) const CHILD_NAME_INDENT: usize = 4; +pub(crate) const LIVE_ROW_PADDING: usize = 7; +pub(crate) const AUDIT_NAME_WIDTH: usize = 50; +pub(crate) const ERROR_PAD: &str = " "; +pub(crate) const ERROR_CONNECTOR_WIDTH: usize = 4; +pub(crate) const UNATTACHED_GROUP: &str = "(unattached)"; +pub(crate) const CLEAR_LINE: &str = "\r\x1b[K"; +pub(crate) const PASS_OUTCOME: &str = "pass"; +pub(crate) const WARN_OUTCOME: &str = "warn"; +pub(crate) const ERROR_OUTCOME: &str = "error"; diff --git a/crates/sqlbuild-core/src/nested_progress/main/audit_group_name.rs b/crates/sqlbuild-core/src/nested_progress/main/audit_group_name.rs new file mode 100644 index 0000000000..9a20999d31 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/main/audit_group_name.rs @@ -0,0 +1,13 @@ +//! The group an audit result row is listed under. + +use crate::build_progress::models::AuditResultText; +use crate::nested_progress::constants::UNATTACHED_GROUP; + +/// The attached target, or `(unattached)`. +#[must_use] +pub fn audit_group_name(result: &AuditResultText) -> String { + match result.attached_target_name.as_deref() { + Some(target) if !target.is_empty() => target.to_owned(), + _ => UNATTACHED_GROUP.to_owned(), + } +} diff --git a/crates/sqlbuild-core/src/nested_progress/main/audit_resource_id.rs b/crates/sqlbuild-core/src/nested_progress/main/audit_resource_id.rs new file mode 100644 index 0000000000..41faf029a7 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/main/audit_resource_id.rs @@ -0,0 +1,17 @@ +//! The execution-protocol identity of one attached audit. + +/// `audit::::`, without empty parts. +#[must_use] +pub fn audit_resource_id( + audit_name: &str, + identity_kind: &str, + (target, column): (Option<&str>, Option<&str>), +) -> String { + ["audit", audit_name, identity_kind] + .into_iter() + .chain(target) + .chain(column) + .filter(|part| !part.is_empty()) + .collect::>() + .join(":") +} diff --git a/crates/sqlbuild-core/src/nested_progress/main/mod.rs b/crates/sqlbuild-core/src/nested_progress/main/mod.rs new file mode 100644 index 0000000000..1f0a78ad20 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/main/mod.rs @@ -0,0 +1,7 @@ +pub mod audit_group_name; +pub mod audit_resource_id; +pub mod render_audit_aggregate; +pub mod render_audit_row; +pub mod render_group_header; +pub mod render_nested_item; +pub mod render_nested_spinner_line; diff --git a/crates/sqlbuild-core/src/nested_progress/main/render_audit_aggregate.rs b/crates/sqlbuild-core/src/nested_progress/main/render_audit_aggregate.rs new file mode 100644 index 0000000000..9bf29b7368 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/main/render_audit_aggregate.rs @@ -0,0 +1,13 @@ +//! The live aggregate row of concurrently running audits. + +use crate::nested_progress::constants::CLEAR_LINE; +use crate::nested_progress::models::AuditAggregate; + +/// Running, completed and worker counts on one cleared line. +#[must_use] +pub fn render_audit_aggregate(aggregate: AuditAggregate) -> String { + format!( + "{CLEAR_LINE}{CLEAR_LINE} audit running {} | completed {}/{} | workers {}", + aggregate.running, aggregate.completed, aggregate.total, aggregate.workers + ) +} diff --git a/crates/sqlbuild-core/src/nested_progress/main/render_audit_row.rs b/crates/sqlbuild-core/src/nested_progress/main/render_audit_row.rs new file mode 100644 index 0000000000..3b9430ebeb --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/main/render_audit_row.rs @@ -0,0 +1,39 @@ +//! One completed standalone audit row. + +use crate::build_progress::models::AuditResultText; +use crate::cli_text::models::CliStyle; +use crate::nested_progress::constants::{ + AUDIT_NAME_WIDTH, ERROR_OUTCOME, LABEL_WIDTH, PASS_OUTCOME, WARN_OUTCOME, +}; + +/// The audit name and column, status, failing row count and optional duration. +#[must_use] +pub fn render_audit_row( + result: &AuditResultText, + duration_ms: Option, + style: CliStyle, +) -> String { + let status_text: &str = match result.outcome.as_str() { + PASS_OUTCOME => "PASS", + WARN_OUTCOME => "WARN", + ERROR_OUTCOME => "FAIL", + _ => "INSUFFICIENT", + }; + let name: String = match &result.attached_column_name { + Some(column) => format!("{} ({column})", result.audit_name), + None => result.audit_name.clone(), + }; + let mut detail: String = String::new(); + if result.outcome != PASS_OUTCOME && result.row_count > 0 { + let label: &str = if result.row_count == 1 { "row" } else { "rows" }; + detail = format!(" {} {label}", result.row_count); + } + if let Some(duration_ms) = duration_ms { + detail = format!("{detail} {:.2}s", duration_ms / 1000.0); + } + format!( + " {: String { + let prefix: &str = if first { "" } else { "\n" }; + format!("{prefix}{}\n", style.section(group_name)) +} diff --git a/crates/sqlbuild-core/src/nested_progress/main/render_nested_item.rs b/crates/sqlbuild-core/src/nested_progress/main/render_nested_item.rs new file mode 100644 index 0000000000..24e338ca69 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/main/render_nested_item.rs @@ -0,0 +1,59 @@ +//! A completed grouped item with its tree of child rows and inline error. + +use crate::cli_text::main::format_inline_error_lines::format_inline_error_lines; +use crate::cli_text::main::format_tree_connector::format_tree_connector; +use crate::cli_text::models::CliStyle; +use crate::nested_progress::constants::{ + CHILD_NAME_INDENT, ERROR_CONNECTOR_WIDTH, ERROR_PAD, LABEL_WIDTH, +}; +use crate::nested_progress::models::{NestedItem, NestedLayout}; + +/// The item row, one tree row per child, and the error as the last tree child. +#[must_use] +pub fn render_nested_item(item: &NestedItem, layout: &NestedLayout, style: CliStyle) -> String { + let width: usize = layout.name_width; + let mut text: String = format!( + " {: = format_inline_error_lines( + ( + item.error.code.as_deref(), + message, + item.error.help.as_deref(), + ), + content_width, + style, + ); + let connector: String = format_tree_connector(style, true); + for (index, line) in lines.iter().enumerate() { + if index == 0 { + text.push_str(&format!("{ERROR_PAD}{connector} {label}{line}\n")); + } else { + text.push_str(&format!("{}{line}\n", " ".repeat(prefix_width))); + } + } + text +} diff --git a/crates/sqlbuild-core/src/nested_progress/main/render_nested_spinner_line.rs b/crates/sqlbuild-core/src/nested_progress/main/render_nested_spinner_line.rs new file mode 100644 index 0000000000..5b8a8d55e7 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/main/render_nested_spinner_line.rs @@ -0,0 +1,36 @@ +//! The single live row naming every running item. + +use crate::cli_text::models::CliStyle; +use crate::nested_progress::constants::{CLEAR_LINE, LABEL_WIDTH, LIVE_ROW_PADDING}; +use crate::nested_progress::models::NestedLayout; + +/// The running item, or a count and list of running items, cut to the terminal. +#[must_use] +pub fn render_nested_spinner_line( + running: &[String], + frame: &str, + layout: &NestedLayout, + style: CliStyle, +) -> String { + let text: String = match running { + [only] => only.clone(), + _ => format!("{} running: {}", running.len(), running.join(", ")), + }; + let available: i64 = layout.columns as i64 - (LABEL_WIDTH + LIVE_ROW_PADDING) as i64; + let width: usize = (layout.name_width as i64).min(available).max(1) as usize; + let name: String = if text.chars().count() <= width { + text + } else { + format!( + "{}...", + text.chars() + .take(width.saturating_sub(3)) + .collect::() + ) + }; + format!( + "{CLEAR_LINE} {:, +} + +/// The row label, name column width and terminal width of one command's rows. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct NestedLayout { + pub label: String, + pub name_width: usize, + pub columns: usize, +} + +/// Aggregate worker state of concurrently running audits. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct AuditAggregate { + pub running: usize, + pub completed: usize, + pub total: usize, + pub workers: i64, +} diff --git a/crates/sqlbuild-core/src/nested_progress/tests.rs b/crates/sqlbuild-core/src/nested_progress/tests.rs new file mode 100644 index 0000000000..718cf08381 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_nested_progress.rs"] +mod nested_progress; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/nested_progress/tests/helpers.rs b/crates/sqlbuild-core/src/nested_progress/tests/helpers.rs new file mode 100644 index 0000000000..00b9ee32fa --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/tests/helpers.rs @@ -0,0 +1,9 @@ +use crate::nested_progress::models::NestedLayout; + +pub(super) fn layout(columns: usize) -> NestedLayout { + NestedLayout { + label: "test".to_owned(), + name_width: 20, + columns, + } +} diff --git a/crates/sqlbuild-core/src/nested_progress/tests/test_nested_progress.rs b/crates/sqlbuild-core/src/nested_progress/tests/test_nested_progress.rs new file mode 100644 index 0000000000..73b8a77bf5 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/tests/test_nested_progress.rs @@ -0,0 +1,87 @@ +use crate::build_progress::models::ErrorText; +use crate::cli_text::models::CliStyle; +use crate::nested_progress::main::render_nested_item::render_nested_item; +use crate::nested_progress::main::render_nested_spinner_line::render_nested_spinner_line; +use crate::nested_progress::models::{NestedChildRow, NestedItem}; +use crate::nested_progress::tests::helpers::layout; +use crate::nested_progress::tests::test_types::{NestedItemTestCase, SpinnerTestCase}; + +const PLAIN: CliStyle = CliStyle { use_color: false }; + +#[test] +fn given_completed_item_when_rendering_then_draws_children_and_error_as_tree() { + let test_cases = [ + NestedItemTestCase { + description: "the last child closes the tree", + item: NestedItem { + item_name: "orders_test".to_owned(), + status_text: "PASS".to_owned(), + detail: " 0.10s".to_owned(), + error: ErrorText::default(), + child_rows: vec![NestedChildRow { + label: "expect".to_owned(), + name: "expected orders".to_owned(), + status_text: "PASS".to_owned(), + detail: String::new(), + }], + }, + expected_text: " test orders_test PASS 0.10s\n \u{2514}\u{2500}\u{2500} expect expected orders PASS\n", + }, + NestedItemTestCase { + description: "an error is the final tree child", + item: NestedItem { + item_name: "orders_test".to_owned(), + status_text: "ERROR".to_owned(), + detail: String::new(), + error: ErrorText { + code: Some("T1".to_owned()), + message: Some("broken".to_owned()), + help: None, + }, + child_rows: Vec::new(), + }, + expected_text: " test orders_test ERROR\n \u{2514}\u{2500}\u{2500} error [T1] broken\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_nested_item(&test_case.item, &layout(80), PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_running_items_when_drawing_spinner_then_lists_and_truncates() { + let test_cases = [ + SpinnerTestCase { + description: "one running item is shown by name", + running: vec!["orders_test".to_owned()], + columns: 80, + expected_text: "\r\x1b[K test orders_test \u{280b}", + }, + SpinnerTestCase { + description: "several running items are counted and cut to the terminal", + running: vec!["orders_test".to_owned(), "customers_test".to_owned()], + columns: 30, + expected_text: "\r\x1b[K test 2 running:... \u{280b}", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_nested_spinner_line( + &test_case.running, + "\u{280b}", + &layout(test_case.columns), + PLAIN + ), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/nested_progress/tests/test_types.rs b/crates/sqlbuild-core/src/nested_progress/tests/test_types.rs new file mode 100644 index 0000000000..41d8d56d84 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/tests/test_types.rs @@ -0,0 +1,14 @@ +use crate::nested_progress::models::NestedItem; + +pub(super) struct NestedItemTestCase { + pub(super) description: &'static str, + pub(super) item: NestedItem, + pub(super) expected_text: &'static str, +} + +pub(super) struct SpinnerTestCase { + pub(super) description: &'static str, + pub(super) running: Vec, + pub(super) columns: usize, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index 4f9d85d50d..1489665b2c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -20,7 +20,6 @@ const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.exte const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main._static_command"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const AUDIT_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._audit"; -const PROGRESS_REPORTER_MODULE: &str = "sqlbuild.cli.progress.classes.audit_progress_reporter"; const COST_CONTEXT_MODULE: &str = "sqlbuild.cost.classes.cost_context"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const PROJECTION_MODULE: &str = "sqlbuild.executor.auditing.main._current_audit_result_projection"; @@ -196,7 +195,7 @@ fn prepare_run<'py>( let stream: Bound<'py, PyAny> = invocation.getattr("progress_stream")?; let use_color: Bound<'py, PyAny> = invocation.getattr("use_color")?; let progress: Bound<'py, PyAny> = call_with( - &imported(py, PROGRESS_REPORTER_MODULE, "AuditProgressReporter")?, + &imported(py, NATIVE_MODULE, "AuditProgressReporter")?, &[ ("entries", entries.clone()), ("worker_limit", PyInt::new(py, worker_count).into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs new file mode 100644 index 0000000000..9724d0d04e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs @@ -0,0 +1,304 @@ +//! Aggregate live state and plan-ordered result rows of standalone concurrent audits. + +use std::collections::HashMap; +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::build_progress::models::AuditResultText; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::nested_progress::main::audit_group_name::audit_group_name; +use sqlbuild_core::nested_progress::main::audit_resource_id::audit_resource_id; +use sqlbuild_core::nested_progress::main::render_audit_aggregate::render_audit_aggregate; +use sqlbuild_core::nested_progress::main::render_audit_row::render_audit_row; +use sqlbuild_core::nested_progress::main::render_group_header::render_group_header; +use sqlbuild_core::nested_progress::models::AuditAggregate; + +use crate::bindings::_helpers::build_progress::result_text::{audit_text, optional, text}; +use crate::bindings::_helpers::build_progress::terminal::{ + CLEAR_LINE, HIDE_CURSOR, SHOW_CURSOR, claim, consume_terminal, flush, projector, release, + shared_lines, tty_flag, write, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, within}; + +/// The identity an audit plan entry or result reports to the progress projector. +fn resource_id(value: &Bound<'_, PyAny>, name_field: &str) -> PyResult { + let kind: Option = match optional(value, "attached_target_kind")? { + Some(kind) => Some(kind), + None => Some(text(value, "attachment_kind")?), + }; + Ok(audit_resource_id( + &text(value, name_field)?, + kind.as_deref().unwrap_or(""), + ( + optional(value, "attached_target_name")?.as_deref(), + optional(value, "attached_column_name")?.as_deref(), + ), + )) +} + +/// Mutable reporter state; only held briefly and never across Python calls. +#[derive(Default)] +struct ReporterState { + completed_by_index: HashMap>>, + next_index: usize, + running: usize, + completed: usize, + current_group: Option, + closed: bool, + cursor_hidden: bool, + on_result: Option>, +} + +/// Render aggregate live state and flush completed audit rows in plan order. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct AuditProgressReporter { + stream: Py, + lines: Py, + lock: Py, + style: CliStyle, + is_tty: bool, + projector: Option>, + indexes: HashMap, + total: usize, + worker_limit: i64, + state: Mutex, +} + +impl AuditProgressReporter { + fn state(&self) -> MutexGuard<'_, ReporterState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn index_of(&self, identity: String) -> PyResult { + self.indexes + .get(&identity) + .copied() + .ok_or_else(|| PyKeyError::new_err(identity)) + } + + fn render_aggregate(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + let (closed, cursor_hidden, running, completed) = { + let state = reporter.state(); + ( + state.closed, + state.cursor_hidden, + state.running, + state.completed, + ) + }; + if !reporter.is_tty || closed { + return Ok(()); + } + let stream: &Bound<'_, PyAny> = reporter.stream.bind(py); + claim(reporter.lines.bind(py), stream, slf.as_any())?; + if !cursor_hidden { + write(stream, HIDE_CURSOR)?; + reporter.state().cursor_hidden = true; + } + write( + stream, + &render_audit_aggregate(AuditAggregate { + running, + completed, + total: reporter.total, + workers: reporter.worker_limit, + }), + )?; + flush(stream) + } + + fn flush_ready(&self, py: Python<'_>) -> PyResult<()> { + loop { + let ready: Option> = { + let mut state = self.state(); + let index: usize = state.next_index; + let Some(result) = state.completed_by_index.remove(&index) else { + return Ok(()); + }; + state.next_index += 1; + result + }; + if let Some(result) = ready { + self.write_result(py, result.bind(py))?; + } + } + } + + fn write_result(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + if self.is_tty { + write(stream, CLEAR_LINE)?; + } + let audit: AuditResultText = audit_text(result)?; + let group: String = audit_group_name(&audit); + let previous: Option = self.state().current_group.clone(); + if previous.as_ref() != Some(&group) { + write( + stream, + &render_group_header(&group, previous.is_none(), self.style), + )?; + self.state().current_group = Some(group); + } + let duration_ms: Option = match &self.projector { + Some(projector) => consume_terminal( + projector.bind(py), + &audit.audit_name, + Some(&resource_id(result, "audit_name")?), + )?, + None => None, + }; + write(stream, &render_audit_row(&audit, duration_ms, self.style))?; + flush(stream)?; + let callback: Option> = self + .state() + .on_result + .as_ref() + .map(|callback| callback.clone_ref(py)); + if let Some(callback) = callback { + callback.bind(py).call1((result,))?; + } + Ok(()) + } + + fn locked(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { + within(self.lock.bind(py), |_| body()) + } +} + +#[pymethods] +impl AuditProgressReporter { + #[new] + #[pyo3(signature = (**options))] + fn new(py: Python<'_>, options: Option>) -> PyResult { + let arguments = KeywordArguments::read( + py, + "AuditProgressReporter", + options, + (&["entries", "worker_limit", "stream", "use_color"], &[]), + )?; + let stream: Bound<'_, PyAny> = arguments.value("stream")?; + let lines: Bound<'_, PyAny> = shared_lines(py)?; + let mut indexes: HashMap = HashMap::new(); + let mut total: usize = 0; + for entry in arguments.value("entries")?.try_iter()? { + indexes.insert(resource_id(&entry?, "name")?, total); + total += 1; + } + Ok(Self { + is_tty: tty_flag(&stream)?, + stream: stream.unbind(), + lock: lines.getattr("lock")?.unbind(), + lines: lines.unbind(), + style: CliStyle { + use_color: arguments.value("use_color")?.is_truthy()?, + }, + projector: projector(py)?, + indexes, + total, + worker_limit: arguments.value("worker_limit")?.extract()?, + state: Mutex::new(ReporterState::default()), + }) + } + + /// Install plan-ordered result enrichment before dispatch starts. + fn set_result_callback(&self, callback: Py) { + self.state().on_result = Some(callback); + } + + /// Record a worker start without mutating one-item spinner state. + fn on_item_start(slf: &Bound<'_, Self>, entry: &Bound<'_, PyAny>) -> PyResult<()> { + let _ = entry; + slf.get().locked(slf.py(), || { + slf.get().state().running += 1; + Self::render_aggregate(slf) + }) + } + + /// Flush every newly contiguous plan-order result row. + fn on_item_complete(slf: &Bound<'_, Self>, result: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + reporter.locked(py, || { + let index: usize = reporter.index_of(resource_id(result, "audit_name")?)?; + reporter + .state() + .completed_by_index + .insert(index, Some(result.clone().unbind())); + reporter.flush_ready(py)?; + Self::render_aggregate(slf) + }) + } + + /// Update aggregate state immediately without projecting a result row. + fn on_item_physical_complete(slf: &Bound<'_, Self>, result: &Bound<'_, PyAny>) -> PyResult<()> { + let _ = result; + slf.get().locked(slf.py(), || { + { + let mut state = slf.get().state(); + state.running = state.running.saturating_sub(1); + state.completed += 1; + } + Self::render_aggregate(slf) + }) + } + + /// Advance aggregate and ordering state for a fatal audit exception. + fn on_item_error(slf: &Bound<'_, Self>, entry: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + reporter.locked(py, || { + let index: usize = reporter.index_of(resource_id(entry, "name")?)?; + { + let mut state = reporter.state(); + state.running = state.running.saturating_sub(1); + state.completed += 1; + state.completed_by_index.insert(index, None); + } + reporter.flush_ready(py)?; + Self::render_aggregate(slf) + }) + } + + /// Restore the terminal after the final aggregate update. + fn close(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + reporter.locked(py, || { + reporter.state().closed = true; + release(reporter.lines.bind(py), slf.as_any())?; + if !reporter.is_tty { + return Ok(()); + } + let stream: &Bound<'_, PyAny> = reporter.stream.bind(py); + write(stream, CLEAR_LINE)?; + let hidden: bool = std::mem::take(&mut reporter.state().cursor_hidden); + if hidden { + write(stream, SHOW_CURSOR)?; + } + flush(stream) + }) + } + + /// Erase the aggregate row so a persistent line can take its place. + fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, CLEAR_LINE)?; + flush(stream) + }) + } + + /// Draw the aggregate row again below persistent output. + fn redraw_transient_line(slf: &Bound<'_, Self>) -> PyResult<()> { + slf.get().locked(slf.py(), || Self::render_aggregate(slf)) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs index 2090874a94..a316c0c55c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs @@ -1,3 +1,6 @@ +pub(crate) mod audit_progress; pub(crate) mod footer; +pub(crate) mod nested_progress; pub(crate) mod registration; pub(crate) mod result_text; +pub(crate) mod terminal; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs new file mode 100644 index 0000000000..fc3fbb5177 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs @@ -0,0 +1,392 @@ +//! One live spinner row for all running items and grouped completed rows of `sqb test`. + +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::build_progress::models::ErrorText; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::nested_progress::main::render_group_header::render_group_header; +use sqlbuild_core::nested_progress::main::render_nested_item::render_nested_item; +use sqlbuild_core::nested_progress::main::render_nested_spinner_line::render_nested_spinner_line; +use sqlbuild_core::nested_progress::models::{NestedChildRow, NestedItem, NestedLayout}; + +use crate::bindings::_helpers::build_progress::result_text::text; +use crate::bindings::_helpers::build_progress::terminal::{ + CLEAR_LINE, HIDE_CURSOR, SHOW_CURSOR, SPINNER_FRAMES, SPINNER_TICK_SECONDS, claim, + consume_terminal, flush, join_spinner, projector, release, shared_lines, start_spinner, + terminal_columns, tty_flag, write, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, within}; + +const MIN_NAME_WIDTH: usize = 50; +const ITEM_OPTIONS: (&[&str], &[&str]) = ( + &["group_name", "item_name"], + &[ + "status_text", + "detail", + "error_code", + "error_help", + "error_message", + "child_rows", + "canonical_resource_name", + "canonical_resource_id", + ], +); + +/// Mutable reporter state; only held briefly and never across Python calls. +#[derive(Default)] +struct ReporterState { + printed_group: Option, + running: Vec<(String, String)>, + frame: usize, + line_active: bool, + cursor_hidden: bool, + stop_event: Option>, + thread: Option>, +} + +/// Render one live spinner row for all running items and grouped completed rows. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NestedCommandProgressCallbacks { + stream: Py, + lines: Py, + lock: Py, + label: String, + name_width: usize, + style: CliStyle, + is_tty: bool, + sequential: bool, + projector: Option>, + state: Mutex, +} + +fn optional_argument(arguments: &KeywordArguments<'_>, name: &str) -> PyResult> { + if !arguments.passed(name)? { + return Ok(None); + } + let value: Bound<'_, PyAny> = arguments.value(name)?; + if value.is_none() { + return Ok(None); + } + value.extract().map(Some) +} + +fn child_rows(arguments: &KeywordArguments<'_>) -> PyResult> { + if !arguments.passed("child_rows")? { + return Ok(Vec::new()); + } + let mut rows: Vec = Vec::new(); + for row in arguments.value("child_rows")?.try_iter()? { + let row: Bound<'_, PyAny> = row?; + rows.push(NestedChildRow { + label: text(&row, "label")?, + name: text(&row, "name")?, + status_text: text(&row, "status_text")?, + detail: text(&row, "detail")?, + }); + } + Ok(rows) +} + +impl NestedCommandProgressCallbacks { + fn state(&self) -> MutexGuard<'_, ReporterState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn locked(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { + within(self.lock.bind(py), |_| body()) + } + + fn layout(&self, py: Python<'_>) -> PyResult { + Ok(NestedLayout { + label: self.label.clone(), + name_width: self.name_width, + columns: terminal_columns(py)?, + }) + } + + fn draw_spinner_line(&self, py: Python<'_>) -> PyResult<()> { + let (running, frame) = { + let mut state = self.state(); + let frame: usize = state.frame; + state.frame = (frame + 1) % SPINNER_FRAMES.len(); + let running: Vec = state.running.iter().map(|(item, _)| item.clone()).collect(); + (running, frame) + }; + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write( + stream, + &render_nested_spinner_line( + &running, + SPINNER_FRAMES[frame], + &self.layout(py)?, + self.style, + ), + )?; + flush(stream) + } + + fn clear_live_row(&self, py: Python<'_>) -> PyResult<()> { + if self.is_tty && self.state().line_active { + write(self.stream.bind(py), CLEAR_LINE)?; + } + Ok(()) + } + + fn write_group_header(&self, py: Python<'_>, group: &str) -> PyResult<()> { + let previous: Option = self.state().printed_group.clone(); + if previous.as_deref() == Some(group) { + return Ok(()); + } + self.state().printed_group = Some(group.to_owned()); + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write( + stream, + &render_group_header(group, previous.is_none(), self.style), + )?; + flush(stream) + } + + fn set_cursor(&self, py: Python<'_>, hidden: bool) -> PyResult<()> { + if self.state().cursor_hidden == hidden { + return Ok(()); + } + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, if hidden { HIDE_CURSOR } else { SHOW_CURSOR })?; + flush(stream)?; + self.state().cursor_hidden = hidden; + Ok(()) + } + + /// Redraw the live row for remaining items, or retire it; return a thread to join. + fn refresh_live_row(slf: &Bound<'_, Self>) -> PyResult>> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + if !reporter.is_tty { + return Ok(None); + } + if !reporter.state().running.is_empty() { + reporter.draw_spinner_line(py)?; + return Ok(None); + } + reporter.state().line_active = false; + release(reporter.lines.bind(py), slf.as_any())?; + reporter.set_cursor(py, false)?; + let (event, thread) = { + let mut state = reporter.state(); + (state.stop_event.take(), state.thread.take()) + }; + if let Some(event) = event { + event.bind(py).call_method0("set")?; + } + Ok(thread) + } +} + +#[pymethods] +impl NestedCommandProgressCallbacks { + #[new] + #[pyo3(signature = (**options))] + fn new(py: Python<'_>, options: Option>) -> PyResult { + let arguments = KeywordArguments::read( + py, + "NestedCommandProgressCallbacks", + options, + ( + &["total", "label", "stream", "use_color"], + &["name_width", "concurrency"], + ), + )?; + let stream: Bound<'_, PyAny> = arguments.value("stream")?; + let lines: Bound<'_, PyAny> = shared_lines(py)?; + let name_width: usize = if arguments.passed("name_width")? { + arguments.value("name_width")?.extract()? + } else { + MIN_NAME_WIDTH + }; + let concurrency: i64 = if arguments.passed("concurrency")? { + arguments.value("concurrency")?.extract()? + } else { + 1 + }; + Ok(Self { + is_tty: tty_flag(&stream)?, + stream: stream.unbind(), + lock: lines.getattr("lock")?.unbind(), + lines: lines.unbind(), + label: arguments.value("label")?.extract()?, + name_width: name_width.max(MIN_NAME_WIDTH), + style: CliStyle { + use_color: arguments.value("use_color")?.is_truthy()?, + }, + sequential: concurrency <= 1, + projector: projector(py)?, + state: Mutex::new(ReporterState::default()), + }) + } + + #[pyo3(signature = (**options))] + fn on_item_start(slf: &Bound<'_, Self>, options: Option>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + let arguments = KeywordArguments::read( + py, + "on_item_start", + options, + (&["group_name", "item_name"], &["canonical_resource_name"]), + )?; + let group: String = arguments.value("group_name")?.extract()?; + let item: String = arguments.value("item_name")?.extract()?; + if let Some(projector) = &reporter.projector { + let canonical: Option = + optional_argument(&arguments, "canonical_resource_name")?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item( + "resource_name", + canonical + .filter(|name| !name.is_empty()) + .unwrap_or(item.clone()), + )?; + projector + .bind(py) + .call_method("expect_resource_enrichment", (), Some(&options))?; + } + reporter.locked(py, || { + if reporter.sequential && reporter.state().running.is_empty() { + reporter.clear_live_row(py)?; + reporter.write_group_header(py, &group)?; + } + { + let mut state = reporter.state(); + match state.running.iter_mut().find(|(name, _)| *name == item) { + Some(entry) => entry.1 = group, + None => state.running.push((item, group)), + } + } + if !reporter.is_tty { + return Ok(()); + } + reporter.set_cursor(py, true)?; + reporter.state().line_active = true; + claim( + reporter.lines.bind(py), + reporter.stream.bind(py), + slf.as_any(), + )?; + reporter.draw_spinner_line(py)?; + if reporter.state().thread.is_none() { + let (event, thread) = start_spinner(slf.as_any())?; + let mut state = reporter.state(); + state.stop_event = Some(event.unbind()); + state.thread = Some(thread.unbind()); + } + Ok(()) + }) + } + + #[pyo3(signature = (**options))] + fn on_item_complete(slf: &Bound<'_, Self>, options: Option>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + let arguments = KeywordArguments::read(py, "on_item_complete", options, ITEM_OPTIONS)?; + let group: String = arguments.value("group_name")?.extract()?; + let item_name: String = arguments.value("item_name")?.extract()?; + let mut detail: String = optional_argument(&arguments, "detail")?.unwrap_or_default(); + let duration_ms: Option = match &reporter.projector { + Some(projector) => { + let canonical: Option = + optional_argument(&arguments, "canonical_resource_name")?; + consume_terminal( + projector.bind(py), + &canonical + .filter(|name| !name.is_empty()) + .unwrap_or(item_name.clone()), + optional_argument(&arguments, "canonical_resource_id")?.as_deref(), + )? + } + None => None, + }; + let stopped: Option> = reporter.locked(py, || { + let item_group: String = { + let mut state = reporter.state(); + match state + .running + .iter() + .position(|(name, _)| *name == item_name) + { + Some(index) => state.running.remove(index).1, + None => group, + } + }; + if reporter.projector.is_none() || duration_ms.is_some() { + if let Some(duration_ms) = duration_ms { + detail = format!("{detail} {:.2}s", duration_ms / 1000.0); + } + let item: NestedItem = NestedItem { + item_name, + status_text: optional_argument(&arguments, "status_text")?.unwrap_or_default(), + detail, + error: ErrorText { + code: optional_argument(&arguments, "error_code")?, + message: optional_argument(&arguments, "error_message")?, + help: optional_argument(&arguments, "error_help")?, + }, + child_rows: child_rows(&arguments)?, + }; + reporter.clear_live_row(py)?; + reporter.write_group_header(py, &item_group)?; + write( + reporter.stream.bind(py), + &render_nested_item(&item, &reporter.layout(py)?, reporter.style), + )?; + } + let stopped: Option> = Self::refresh_live_row(slf)?; + flush(reporter.stream.bind(py))?; + Ok(stopped) + })?; + if let Some(thread) = stopped { + join_spinner(thread.bind(py))?; + } + Ok(()) + } + + /// Erase the live spinner row so a persistent line can take its place. + fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, CLEAR_LINE)?; + flush(stream) + }) + } + + /// Draw the live spinner row again below persistent output. + fn redraw_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || { + if self.state().line_active { + self.draw_spinner_line(py)?; + } + Ok(()) + }) + } + + fn _spin_until_stopped(&self, py: Python<'_>, stop_event: &Bound<'_, PyAny>) -> PyResult<()> { + while !stop_event + .call_method1("wait", (SPINNER_TICK_SECONDS,))? + .is_truthy()? + { + self.locked(py, || { + if !stop_event.call_method0("is_set")?.is_truthy()? && self.state().line_active { + self.draw_spinner_line(py)?; + } + Ok(()) + })?; + } + Ok(()) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs index 52c90493f2..dd7135ce24 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs @@ -1,9 +1,11 @@ -//! Register the native build progress reporter and footer. +//! Register the native build, audit and test progress reporters and the build footer. use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::build_progress::footer; +use crate::bindings::_helpers::build_progress::{audit_progress, footer, nested_progress}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - footer::register(module) + audit_progress::register(module)?; + footer::register(module)?; + nested_progress::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs new file mode 100644 index 0000000000..65a1bb16ac --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs @@ -0,0 +1,118 @@ +//! Terminal width, the shared transient line coordinator and spinner threads of progress reporters. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const COLUMNS_MODULE: &str = "sqlbuild.presentation.main.terminal_columns"; +const LINES_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; +const SPINNER_JOIN_SECONDS: f64 = 0.2; +pub(crate) const SPINNER_TICK_SECONDS: f64 = 0.1; +pub(crate) const CLEAR_LINE: &str = "\r\u{1b}[K"; +pub(crate) const HIDE_CURSOR: &str = "\u{1b}[?25l"; +pub(crate) const SHOW_CURSOR: &str = "\u{1b}[?25h"; +pub(crate) const SPINNER_FRAMES: [&str; 10] = [ + "\u{280b}", "\u{2819}", "\u{2839}", "\u{2838}", "\u{283c}", "\u{2834}", "\u{2826}", "\u{2827}", + "\u{2807}", "\u{280f}", +]; + +/// The current terminal width. +pub(crate) fn terminal_columns(py: Python<'_>) -> PyResult { + let columns: i64 = imported(py, COLUMNS_MODULE, "terminal_columns")? + .call0()? + .extract()?; + Ok(usize::try_from(columns).unwrap_or(0)) +} + +/// The process-wide coordinator of the one live transient line. +pub(crate) fn shared_lines(py: Python<'_>) -> PyResult> { + imported(py, LINES_MODULE, "shared_transient_line_coordinator")?.call0() +} + +/// Whether `stream` reports a terminal. +pub(crate) fn tty_flag(stream: &Bound<'_, PyAny>) -> PyResult { + if !stream.hasattr("isatty")? { + return Ok(false); + } + stream.call_method0("isatty")?.is_truthy() +} + +/// The current native progress projector, if a projection is active. +pub(crate) fn projector(py: Python<'_>) -> PyResult>> { + let projector: Bound<'_, PyAny> = + imported(py, "sqlbuild._native", "current_native_progress_projector")?.call0()?; + Ok((!projector.is_none()).then(|| projector.unbind())) +} + +/// The projector's canonical duration of a finished resource, `None` while it is pending. +pub(crate) fn consume_terminal( + projector: &Bound<'_, PyAny>, + resource_name: &str, + resource_id: Option<&str>, +) -> PyResult> { + let options: Bound<'_, PyDict> = PyDict::new(projector.py()); + options.set_item("resource_name", resource_name)?; + options.set_item("resource_id", resource_id)?; + projector + .call_method("consume_resource_terminal", (), Some(&options))? + .extract() +} + +/// Claim or release the live line for `owner`. +pub(crate) fn claim( + lines: &Bound<'_, PyAny>, + stream: &Bound<'_, PyAny>, + owner: &Bound<'_, PyAny>, +) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(lines.py()); + options.set_item("stream", stream)?; + options.set_item("owner", owner)?; + lines.call_method("claim", (), Some(&options))?; + Ok(()) +} + +pub(crate) fn release(lines: &Bound<'_, PyAny>, owner: &Bound<'_, PyAny>) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(lines.py()); + options.set_item("owner", owner)?; + lines.call_method("release", (), Some(&options))?; + Ok(()) +} + +pub(crate) fn write(stream: &Bound<'_, PyAny>, text: &str) -> PyResult<()> { + stream.call_method1("write", (text,))?; + Ok(()) +} + +pub(crate) fn flush(stream: &Bound<'_, PyAny>) -> PyResult<()> { + stream.call_method0("flush")?; + Ok(()) +} + +/// Start a daemon thread running `owner._spin_until_stopped(event)`; return the event and thread. +pub(crate) fn start_spinner<'py>( + owner: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = owner.py(); + let event: Bound<'py, PyAny> = imported(py, "threading", "Event")?.call0()?; + let thread: Bound<'py, PyAny> = call_with( + &imported(py, "threading", "Thread")?, + &[ + ("target", owner.getattr("_spin_until_stopped")?), + ("args", PyTuple::new(py, [&event])?.into_any()), + ("daemon", PyBool::new(py, true).to_owned().into_any()), + ], + )?; + thread.call_method0("start")?; + Ok((event, thread)) +} + +/// Join a stopped spinner thread briefly if it is still alive. +pub(crate) fn join_spinner(thread: &Bound<'_, PyAny>) -> PyResult<()> { + if thread.call_method0("is_alive")?.is_truthy()? { + let options: Bound<'_, PyDict> = PyDict::new(thread.py()); + options.set_item("timeout", SPINNER_JOIN_SECONDS)?; + thread.call_method("join", (), Some(&options))?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index beb0610944..1576b1b915 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -36,8 +36,6 @@ const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main._static_com const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const TEST_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._sql_test"; const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; -const NESTED_PROGRESS_MODULE: &str = - "sqlbuild.cli.progress.classes.nested_command_progress_callbacks"; const PROGRESS_MODELS: &str = "sqlbuild.cli.progress.models"; const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; const EXECUTOR_PIPELINE_MODELS: &str = "sqlbuild.executor.pipeline.models"; @@ -486,7 +484,7 @@ fn run_tests<'py>( } let worker_count: i64 = concurrency.min(i64::try_from(test_count).unwrap_or(i64::MAX)); let progress: Bound<'py, PyAny> = call_with( - &imported(py, NESTED_PROGRESS_MODULE, "NestedCommandProgressCallbacks")?, + &imported(py, NATIVE_MODULE, "NestedCommandProgressCallbacks")?, &[ ("total", PyInt::new(py, test_count).into_any()), ("label", text(py, "test")), diff --git a/python-boundary.toml b/python-boundary.toml index e14c07366d..9c2976c0aa 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 62087 +remaining_lines = 61597 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 70863c345a..5d5b012e9b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3191,6 +3191,55 @@ class LoadProgressReporter: def clear_transient_line(self) -> None: ... def redraw_transient_line(self) -> None: ... +class AuditProgressReporter: + def __init__( + self, + *, + entries: tuple[AuditPlanEntry, ...], + worker_limit: int, + stream: TextIO, + use_color: bool, + ) -> None: ... + def set_result_callback(self, callback: Callable[[AuditExecutionResult], None]) -> None: ... + def on_item_start(self, entry: AuditPlanEntry) -> None: ... + def on_item_complete(self, result: AuditExecutionResult) -> None: ... + def on_item_physical_complete(self, result: AuditExecutionResult) -> None: ... + def on_item_error(self, entry: AuditPlanEntry) -> None: ... + def close(self) -> None: ... + def clear_transient_line(self) -> None: ... + def redraw_transient_line(self) -> None: ... + +class NestedCommandProgressCallbacks: + def __init__( + self, + *, + total: int, + label: str, + stream: TextIO, + use_color: bool, + name_width: int = 50, + concurrency: int = 1, + ) -> None: ... + def on_item_start( + self, *, group_name: str, item_name: str, canonical_resource_name: str | None = None + ) -> None: ... + def on_item_complete( + self, + *, + group_name: str, + item_name: str, + status_text: str, + detail: str = "", + error_code: str | None = None, + error_help: str | None = None, + error_message: str | None = None, + child_rows: tuple[NestedProgressChildRow, ...] = (), + canonical_resource_name: str | None = None, + canonical_resource_id: str | None = None, + ) -> None: ... + def clear_transient_line(self) -> None: ... + def redraw_transient_line(self) -> None: ... + def observe_source_freshness_json( *, adapter: object, diff --git a/src/sqlbuild/cli/progress/classes/__init__.py b/src/sqlbuild/cli/progress/classes/__init__.py deleted file mode 100644 index 055ba47d35..0000000000 --- a/src/sqlbuild/cli/progress/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""CLI progress classes.""" diff --git a/src/sqlbuild/cli/progress/classes/audit_progress_reporter.py b/src/sqlbuild/cli/progress/classes/audit_progress_reporter.py deleted file mode 100644 index b6f2bc1551..0000000000 --- a/src/sqlbuild/cli/progress/classes/audit_progress_reporter.py +++ /dev/null @@ -1,198 +0,0 @@ -"""Coordinator-safe progress reporting for standalone concurrent audits.""" - -from __future__ import annotations - -import threading -from collections.abc import Callable -from typing import TextIO - -from sqlbuild._native import ( - NativeProgressProjector, - current_native_progress_projector, -) -from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.compiler.planner.models import AuditPlanEntry -from sqlbuild.executor.auditing.main.resource_id import audit_resource_id -from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator -from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator - -_LABEL_WIDTH: int = 10 -_NAME_WIDTH: int = 50 - - -class AuditProgressReporter: - """Render aggregate live state and flush completed audit rows in plan order.""" - - def __init__( - self, - *, - entries: tuple[AuditPlanEntry, ...], - worker_limit: int, - stream: TextIO, - use_color: bool, - ) -> None: - self._entries: tuple[AuditPlanEntry, ...] = entries - self._worker_limit: int = worker_limit - self._stream: TextIO = stream - self._lines: TransientLineCoordinator = shared_transient_line_coordinator() - self._style: CliStyle = CliStyle(use_color=use_color) - self._is_tty: bool = hasattr(stream, "isatty") and stream.isatty() - self._projector: NativeProgressProjector | None = current_native_progress_projector() - self._indexes: dict[str, int] = { - _entry_resource_id(entry): index for index, entry in enumerate(entries) - } - self._completed_by_index: dict[int, AuditExecutionResult | None] = {} - self._next_index: int = 0 - self._running: int = 0 - self._completed: int = 0 - self._current_group: str | None = None - self._lock: threading.RLock = self._lines.lock - self._closed: bool = False - self._cursor_hidden: bool = False - self._on_result: Callable[[AuditExecutionResult], None] | None = None - - def set_result_callback(self, callback: Callable[[AuditExecutionResult], None]) -> None: - """Install plan-ordered result enrichment before dispatch starts.""" - - self._on_result = callback - - def on_item_start(self, entry: AuditPlanEntry) -> None: - """Record a worker start without mutating one-item spinner state.""" - - with self._lock: - self._running += 1 - self._render_aggregate() - - def on_item_complete(self, result: AuditExecutionResult) -> None: - """Flush every newly contiguous plan-order result row.""" - - with self._lock: - self._completed_by_index[self._indexes[_result_resource_id(result)]] = result - self._flush_ready() - self._render_aggregate() - - def on_item_physical_complete(self, result: AuditExecutionResult) -> None: - """Update aggregate state immediately without projecting a result row.""" - - del result - with self._lock: - self._running = max(0, self._running - 1) - self._completed += 1 - self._render_aggregate() - - def on_item_error(self, entry: AuditPlanEntry) -> None: - """Advance aggregate and ordering state for a fatal audit exception.""" - - with self._lock: - self._running = max(0, self._running - 1) - self._completed += 1 - self._completed_by_index[self._indexes[_entry_resource_id(entry)]] = None - self._flush_ready() - self._render_aggregate() - - def close(self) -> None: - """Restore the terminal after the final aggregate update.""" - - with self._lock: - self._closed = True - self._lines.release(owner=self) - if self._is_tty: - self._clear_aggregate() - if self._cursor_hidden: - self._stream.write("\033[?25h") - self._cursor_hidden = False - self._stream.flush() - - def _flush_ready(self) -> None: - while self._next_index in self._completed_by_index: - result: AuditExecutionResult | None = self._completed_by_index.pop(self._next_index) - self._next_index += 1 - if result is not None: - self._write_result(result) - - def _write_result(self, result: AuditExecutionResult) -> None: - if self._is_tty: - self._clear_aggregate() - group_name: str = result.attached_target_name or "(unattached)" - if group_name != self._current_group: - prefix: str = "\n" if self._current_group is not None else "" - self._stream.write(f"{prefix}{self._style.section(group_name)}\n") - self._current_group = group_name - status_text: str = { - AuditOutcome.PASS: "PASS", - AuditOutcome.WARN: "WARN", - AuditOutcome.ERROR: "FAIL", - AuditOutcome.INSUFFICIENT: "INSUFFICIENT", - }[result.outcome] - name: str = result.audit_name - if result.attached_column_name is not None: - name = f"{name} ({result.attached_column_name})" - detail: str = "" - if result.outcome != AuditOutcome.PASS and result.row_count > 0: - row_label: str = "row" if result.row_count == 1 else "rows" - detail = f" {result.row_count} {row_label}" - if self._projector is not None: - duration_ms: float | None = self._projector.consume_resource_terminal( - resource_name=result.audit_name, - resource_id=_result_resource_id(result), - ) - if duration_ms is not None: - detail = f"{detail} {duration_ms / 1000.0:.2f}s" - status: str = self._style.status(status=status_text) - self._stream.write(f" {'audit':<{_LABEL_WIDTH}}{name:<{_NAME_WIDTH}} {status}{detail}\n") - self._stream.flush() - if self._on_result is not None: - self._on_result(result) - - def clear_transient_line(self) -> None: - """Erase the aggregate row so a persistent line can take its place.""" - - with self._lock: - self._clear_aggregate() - self._stream.flush() - - def redraw_transient_line(self) -> None: - """Draw the aggregate row again below persistent output.""" - - with self._lock: - self._render_aggregate() - - def _render_aggregate(self) -> None: - if not self._is_tty or self._closed: - return - self._lines.claim(stream=self._stream, owner=self) - if not self._cursor_hidden: - self._stream.write("\033[?25l") - self._cursor_hidden = True - self._clear_aggregate() - self._stream.write( - "\r\033[K audit " - f"running {self._running} | completed {self._completed}/{len(self._entries)} " - f"| workers {self._worker_limit}" - ) - self._stream.flush() - - def _clear_aggregate(self) -> None: - self._stream.write("\r\033[K") - - -def _entry_resource_id(entry: AuditPlanEntry) -> str: - return audit_resource_id( - audit_name=entry.name, - attachment_kind=entry.attachment_kind, - attached_target_kind=entry.attached_target_kind, - attached_target_name=entry.attached_target_name, - attached_column_name=entry.attached_column_name, - ) - - -def _result_resource_id(result: AuditExecutionResult) -> str: - return audit_resource_id( - audit_name=result.audit_name, - attachment_kind=result.attachment_kind, - attached_target_kind=result.attached_target_kind, - attached_target_name=result.attached_target_name, - attached_column_name=result.attached_column_name, - ) diff --git a/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py b/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py deleted file mode 100644 index 8b6c6ec396..0000000000 --- a/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py +++ /dev/null @@ -1,288 +0,0 @@ -"""Progress rendering for grouped audit and test style commands.""" - -from __future__ import annotations - -import threading -from typing import TextIO - -from sqlbuild._native import ( - NativeProgressProjector, - current_native_progress_projector, -) -from sqlbuild.cli.progress.constants import NESTED_CHILD_NAME_INDENT -from sqlbuild.cli.progress.models import NestedProgressChildRow -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator -from sqlbuild.presentation.main.inline_error_lines import format_inline_error_lines -from sqlbuild.presentation.main.terminal_columns import terminal_columns -from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator -from sqlbuild.presentation.main.tree_connector import tree_connector - -_LABEL_WIDTH: int = 10 -_NAME_WIDTH: int = 50 -_SPINNER_TICK_SECONDS: float = 0.1 -_LIVE_ROW_PADDING: int = 7 -_ACTIVE_SPINNER_FRAMES: tuple[str, ...] = ( - "⠋", - "⠙", - "⠹", - "⠸", - "⠼", - "⠴", - "⠦", - "⠧", - "⠇", - "⠏", -) - - -class NestedCommandProgressCallbacks: - """Render one live spinner row for all running items and grouped completed rows.""" - - def __init__( - self, - *, - total: int, - label: str, - stream: TextIO, - use_color: bool, - name_width: int = _NAME_WIDTH, - concurrency: int = 1, - ) -> None: - self._total: int = total - self._label: str = label - self._stream: TextIO = stream - self._lines: TransientLineCoordinator = shared_transient_line_coordinator() - self._use_color: bool = use_color - self._style: CliStyle = CliStyle(use_color=use_color) - self._name_width: int = max(_NAME_WIDTH, name_width) - self._is_tty: bool = hasattr(stream, "isatty") and stream.isatty() - self._sequential: bool = concurrency <= 1 - self._printed_group: str | None = None - self._running: dict[str, str] = {} - self._spinner_frame_index: int = 0 - self._write_lock: threading.RLock = self._lines.lock - self._spinner_line_active: bool = False - self._spinner_stop_event: threading.Event | None = None - self._spinner_thread: threading.Thread | None = None - self._cursor_hidden: bool = False - self._projector: NativeProgressProjector | None = current_native_progress_projector() - - def on_item_start( - self, - *, - group_name: str, - item_name: str, - canonical_resource_name: str | None = None, - ) -> None: - if self._projector is not None: - self._projector.expect_resource_enrichment( - resource_name=canonical_resource_name or item_name - ) - with self._write_lock: - if self._sequential and not self._running: - self._clear_live_row() - self._write_group_header(group_name) - self._running[item_name] = group_name - if self._is_tty: - self._hide_cursor() - self._spinner_line_active = True - self._lines.claim(stream=self._stream, owner=self) - self._draw_spinner_line() - self._ensure_spinner_loop() - - def on_item_complete( - self, - *, - group_name: str, - item_name: str, - status_text: str, - detail: str = "", - error_code: str | None = None, - error_help: str | None = None, - error_message: str | None = None, - child_rows: tuple[NestedProgressChildRow, ...] = (), - canonical_resource_name: str | None = None, - canonical_resource_id: str | None = None, - ) -> None: - duration_ms: float | None = None - if self._projector is not None: - duration_ms = self._projector.consume_resource_terminal( - resource_name=canonical_resource_name or item_name, - resource_id=canonical_resource_id, - ) - stopped_thread: threading.Thread | None - with self._write_lock: - item_group: str = self._running.pop(item_name, group_name) - if self._projector is None or duration_ms is not None: - if duration_ms is not None: - detail = f"{detail} {duration_ms / 1000.0:.2f}s" - self._clear_live_row() - self._write_group_header(item_group) - self._write_completed_item( - item_name=item_name, - status_text=status_text, - detail=detail, - error_code=error_code, - error_help=error_help, - error_message=error_message, - child_rows=child_rows, - ) - stopped_thread = self._refresh_live_row() - self._stream.flush() - if stopped_thread is not None and stopped_thread.is_alive(): - stopped_thread.join(timeout=0.2) - - def _write_group_header(self, group_name: str) -> None: - if group_name == self._printed_group: - return - prefix: str = "" if self._printed_group is None else "\n" - self._printed_group = group_name - self._stream.write(f"{prefix}{self._style.section(group_name)}\n") - self._stream.flush() - - def _write_completed_item( - self, - *, - item_name: str, - status_text: str, - detail: str, - error_code: str | None, - error_help: str | None, - error_message: str | None, - child_rows: tuple[NestedProgressChildRow, ...], - ) -> None: - status: str = self._style.status(status=status_text) - self._stream.write( - f" {self._label:<{_LABEL_WIDTH}}{item_name:<{self._name_width}} {status}{detail}\n" - ) - child_name_width: int = self._name_width - NESTED_CHILD_NAME_INDENT - child_row: NestedProgressChildRow - for index, child_row in enumerate(child_rows): - child_status: str = self._style.status(status=child_row.status_text) - last: bool = index == len(child_rows) - 1 and error_message is None - connector: str = tree_connector(style=self._style, last=last) - self._stream.write( - f" {connector} {child_row.label:<{_LABEL_WIDTH - 2}}" - f"{child_row.name:<{child_name_width}} " - f"{child_status}{child_row.detail}\n" - ) - if error_message is not None: - self._write_error( - error_code=error_code, - error_message=error_message, - error_help=error_help, - ) - - def _clear_live_row(self) -> None: - if self._is_tty and self._spinner_line_active: - self._stream.write("\r\033[K") - - def _refresh_live_row(self) -> threading.Thread | None: - """Redraw the live row for remaining items, or retire it; return a thread to join.""" - - if not self._is_tty: - return None - if self._running: - self._draw_spinner_line() - return None - self._spinner_line_active = False - self._lines.release(owner=self) - self._show_cursor() - stopped_thread: threading.Thread | None = self._spinner_thread - if self._spinner_stop_event is not None: - self._spinner_stop_event.set() - self._spinner_stop_event = None - self._spinner_thread = None - return stopped_thread - - def _write_error( - self, *, error_code: str | None, error_message: str, error_help: str | None - ) -> None: - pad: str = " " * 6 - connector: str = tree_connector(style=self._style, last=True) - label_width: int = _LABEL_WIDTH - 2 - label: str = self._style.error_muted(f"{'error':<{label_width}}") - plain_prefix_width: int = len(pad) + 4 + label_width - content_width: int = max(terminal_columns() - plain_prefix_width, 1) - lines: list[str] = format_inline_error_lines( - error_code=error_code, - error_message=error_message, - error_help=error_help, - content_width=content_width, - style=self._style, - ) - continuation_pad: str = " " * plain_prefix_width - for index, line in enumerate(lines): - if index == 0: - self._stream.write(f"{pad}{connector} {label}{line}\n") - else: - self._stream.write(f"{continuation_pad}{line}\n") - - def clear_transient_line(self) -> None: - """Erase the live spinner row so a persistent line can take its place.""" - - with self._write_lock: - self._stream.write("\r\033[K") - self._stream.flush() - - def redraw_transient_line(self) -> None: - """Draw the live spinner row again below persistent output.""" - - self._write_spinner_line() - - def _write_spinner_line(self) -> None: - with self._write_lock: - if self._spinner_line_active: - self._draw_spinner_line() - - def _draw_spinner_line(self) -> None: - spinner: str = self._style.status(status=_ACTIVE_SPINNER_FRAMES[self._spinner_frame_index]) - self._spinner_frame_index = (self._spinner_frame_index + 1) % len(_ACTIVE_SPINNER_FRAMES) - running: tuple[str, ...] = tuple(self._running) - text: str = ( - running[0] if len(running) == 1 else f"{len(running)} running: {', '.join(running)}" - ) - name_width: int = max( - min(self._name_width, terminal_columns() - _LABEL_WIDTH - _LIVE_ROW_PADDING), 1 - ) - name: str = text if len(text) <= name_width else f"{text[: max(name_width - 3, 0)]}..." - self._stream.write( - f"\r\033[K {self._label:<{_LABEL_WIDTH}}{name:<{name_width}} {spinner}" - ) - self._stream.flush() - - def _ensure_spinner_loop(self) -> None: - if self._spinner_thread is not None: - return - stop_event: threading.Event = threading.Event() - self._spinner_stop_event = stop_event - spinner_thread: threading.Thread = threading.Thread( - target=self._spin_until_stopped, - args=(stop_event,), - daemon=True, - ) - self._spinner_thread = spinner_thread - spinner_thread.start() - - def _spin_until_stopped(self, stop_event: threading.Event) -> None: - while not stop_event.wait(_SPINNER_TICK_SECONDS): - with self._write_lock: - if not stop_event.is_set() and self._spinner_line_active: - self._draw_spinner_line() - - def _hide_cursor(self) -> None: - if self._cursor_hidden: - return - with self._write_lock: - self._stream.write("\033[?25l") - self._stream.flush() - self._cursor_hidden = True - - def _show_cursor(self) -> None: - if not self._cursor_hidden: - return - with self._write_lock: - self._stream.write("\033[?25h") - self._stream.flush() - self._cursor_hidden = False diff --git a/src/sqlbuild/cli/progress/constants.py b/src/sqlbuild/cli/progress/constants.py deleted file mode 100644 index fb554beaa8..0000000000 --- a/src/sqlbuild/cli/progress/constants.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Shared layout constants for CLI progress rendering.""" - -NESTED_CHILD_NAME_INDENT: int = 4 diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py index 435f92a3c7..9d73059397 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py @@ -10,6 +10,7 @@ import pytest from sqlbuild._native import ( + NestedCommandProgressCallbacks, build_test_expectation_rows, resolve_test_name_width, write_execution_header, @@ -23,9 +24,6 @@ _truncate_name, format_build_footer, ) -from sqlbuild.cli.progress.classes.nested_command_progress_callbacks import ( - NestedCommandProgressCallbacks, -) from sqlbuild.cli.progress.models import AuditDisplayEntry, NestedProgressChildRow from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope from sqlbuild.compiler.planner.models import PlanOutput diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py index 02f9f3bdb6..e3cb4245ec 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py @@ -18,9 +18,9 @@ import pytest +from sqlbuild._native import AuditProgressReporter from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.progress.classes.audit_progress_reporter import AuditProgressReporter from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditOutcome, From 8094e804a9a074442239aa77854282a17d4f4121 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 03:47:52 +0100 Subject: [PATCH 03/40] refactor: port the build progress callbacks to a native class --- .../_helpers/build_command/completion.rs | 3 +- .../_helpers/build_command/execution.rs | 3 +- .../build_progress/build_callbacks.rs | 614 +++++++ .../bindings/_helpers/build_progress/mod.rs | 2 + .../_helpers/build_progress/plan_facts.rs | 120 ++ .../_helpers/build_progress/registration.rs | 5 +- .../_helpers/result_output/projection.rs | 2 +- .../_helpers/sql_test_command/test_command.rs | 2 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 30 +- .../commands/_helpers/dbt/sqlbuild_work.py | 6 +- .../classes/build_progress_callbacks.py | 1580 ----------------- src/sqlbuild/cli/commands/models.py | 1 - src/sqlbuild/cli/progress/models.py | 29 - .../executor/build/classes/build_scheduler.py | 2 +- .../executor/pipeline/_helpers/testing.py | 2 +- .../main/{resource_id.py => _resource_id.py} | 0 .../_helpers/test_module_paths.py | 1 + .../commands/shared/_helpers/_test_types.py | 22 - .../cli/commands/shared/_helpers/helpers.py | 11 - .../commands/shared/_helpers/test_progress.py | 263 +-- .../classes/test_build_sql_test_projection.py | 2 +- 22 files changed, 787 insertions(+), 1915 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs delete mode 100644 src/sqlbuild/cli/commands/classes/build_progress_callbacks.py rename src/sqlbuild/executor/testing/main/{resource_id.py => _resource_id.py} (100%) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index a4ef5802f0..7dd3ddb644 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -25,7 +25,6 @@ const COST_COLLECTION_MODULE: &str = "sqlbuild.cli.commands._helpers.cost.collec const COST_TYPES: &str = "sqlbuild.cost.types"; const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const BUILD_TYPES: &str = "sqlbuild.executor.build.types"; -const CALLBACKS_MODULE: &str = "sqlbuild.cli.commands.classes.build_progress_callbacks"; const RUNTIME_TARGET_MODULE: &str = "sqlbuild.cli.target_artifacts.main._write_runtime_target"; const CHECK_TARGET_MODULE: &str = "sqlbuild.cli.target_artifacts.main._write_python_check_runtime_target"; @@ -360,7 +359,7 @@ pub(crate) fn complete_direct_build<'py>( let cost_seconds: f64 = finish_cost_timing(context, cost_started); let record: Bound<'py, PyAny> = record?; let footer: String = call_with( - &imported(py, CALLBACKS_MODULE, "format_build_footer")?, + &imported(py, NATIVE_MODULE, "format_build_footer")?, &[ ("result", result.clone()), ("elapsed", preparation.callbacks.getattr("elapsed")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index b31ca9c1cf..61bbfd3bce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -17,7 +17,6 @@ use crate::bindings::_helpers::build_command::support::{ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const CALLBACKS_MODULE: &str = "sqlbuild.cli.commands.classes.build_progress_callbacks"; const PARSING_MODULE: &str = "sqlbuild.cli.commands._helpers.input.parsing"; const CURSOR_OVERRIDES_MODULE: &str = "sqlbuild.compiler.planner.main.cursor_window._typed_cursor_overrides"; @@ -207,7 +206,7 @@ pub(crate) fn prepare_build_execution<'py>( let plan: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; let debug: bool = context.request_flag("debug")? || context.request_flag("json_output")?; let callbacks: Bound<'py, PyAny> = call_with( - &imported(py, CALLBACKS_MODULE, "BuildProgressCallbacks")?, + &imported(py, NATIVE_MODULE, "BuildProgressCallbacks")?, &[ ("plan", plan.clone()), ("use_color", boolean(py, context.use_color)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs new file mode 100644 index 0000000000..472ad71d1d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs @@ -0,0 +1,614 @@ +//! Live progress rows, completion rows and runtime diagnostics of `sqb build`. + +use std::collections::HashMap; +use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::time::Instant; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::build_progress::main::query_diagnostic_message::query_diagnostic_message; +use sqlbuild_core::build_progress::main::render_build_spinner_line::render_build_spinner_line; +use sqlbuild_core::build_progress::main::render_failed_test_result::render_failed_test_result; +use sqlbuild_core::build_progress::main::render_model_result::render_model_result; +use sqlbuild_core::build_progress::main::render_resource_result::render_resource_result; +use sqlbuild_core::build_progress::main::runtime_diagnostic_summary::runtime_diagnostic_summary; +use sqlbuild_core::build_progress::main::scheduler_diagnostic_message::scheduler_diagnostic_message; +use sqlbuild_core::build_progress::models::{ + ModelRowFacts, ProgressLayout, ResourceResult, SchedulerStateText, SpinnerRow, StatementText, + TestResultText, +}; +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::bindings::_helpers::build_progress::plan_facts::{BuildPlanFacts, build_plan_facts}; +use crate::bindings::_helpers::build_progress::result_text::{ + function_text, load_text, model_text, optional, seed_text, test_text, text, +}; +use crate::bindings::_helpers::build_progress::terminal::{ + CLEAR_LINE, HIDE_CURSOR, SHOW_CURSOR, SPINNER_FRAMES, SPINNER_TICK_SECONDS, claim, + consume_terminal, flush, join_spinner, projector, release, shared_lines, start_spinner, + terminal_columns, tty_flag, write, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported, within}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const RUNTIME_LOGGER: &str = "sqlbuild.executor.runtime"; +const PASS_OUTCOME: &str = "pass"; +const SCHEDULER_DIAGNOSTIC_LIMIT: usize = 50; +const QUERY_DIAGNOSTIC_LIMIT: usize = 25; + +/// Which executor result type a completion reports. +#[derive(Clone, Copy, PartialEq, Eq)] +enum ResultKind { + Model, + Seed, + Function, + Load, + PythonNode, + SqlTest, + Other, +} + +fn result_kind(result: &Bound<'_, PyAny>) -> PyResult { + for (field, kind) in [ + ("model_name", ResultKind::Model), + ("seed_name", ResultKind::Seed), + ("function_name", ResultKind::Function), + ("loader_name", ResultKind::Load), + ("node_name", ResultKind::PythonNode), + ("test_name", ResultKind::SqlTest), + ] { + if result.hasattr(field)? { + return Ok(kind); + } + } + Ok(ResultKind::Other) +} + +fn sql_test_resource_id(result: &Bound<'_, PyAny>) -> PyResult { + let name: String = text(result, "test_name")?; + let path: Bound<'_, PyAny> = result.getattr("source_path")?; + let block: Bound<'_, PyAny> = result.getattr("block_index")?; + let case: Bound<'_, PyAny> = result.getattr("case_name")?; + if path.is_none() || block.is_none() || case.is_none() { + return Ok(format!("sql_test:{name}")); + } + Ok(format!( + "sql_test:{}:{}:{}", + path.call_method0("as_posix")?.str()?, + block.str()?, + case.str()? + )) +} + +/// The resource name and projector identity of a completion. +fn resource_identity( + result: &Bound<'_, PyAny>, + kind: ResultKind, +) -> PyResult> { + Ok(match kind { + ResultKind::Model => { + let name: String = text(result, "model_name")?; + Some((name.clone(), format!("model:{name}"))) + } + ResultKind::Seed => { + let name: String = text(result, "seed_name")?; + Some((name.clone(), format!("seed:{name}"))) + } + ResultKind::Function => { + let name: String = text(result, "function_name")?; + let kind: String = text(result, "function_kind")?; + Some((name.clone(), format!("{kind}:{name}"))) + } + ResultKind::Load => { + let name: String = text(result, "source_name")?; + Some((name.clone(), format!("source:{name}"))) + } + ResultKind::PythonNode => { + let name: String = text(result, "node_name")?; + let kind: String = text(result, "kind")?; + Some((name.clone(), format!("{kind}:{name}"))) + } + ResultKind::SqlTest => Some((text(result, "test_name")?, sql_test_resource_id(result)?)), + ResultKind::Other => None, + }) +} + +/// Mutable callback state; only held briefly and never across Python calls. +#[derive(Default)] +struct CallbackState { + counter: usize, + node_name: String, + node_kind: String, + sub_message: String, + frame: usize, + line_active: bool, + cursor_hidden: bool, + stop_event: Option>, + thread: Option>, + tests_by_model: HashMap>, + scheduler_count: usize, + scheduler_omitted: usize, + last_scheduler: Option, + query_count: usize, + query_omitted: usize, + last_query: Option, +} + +/// Encapsulates live build progress output state and callbacks. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct BuildProgressCallbacks { + plan: Py, + stream: Py, + lines: Py, + lock: Py, + event_writer: Py, + projector: Option>, + facts: BuildPlanFacts, + style: CliStyle, + verbose: bool, + debug: bool, + is_tty: bool, + started: Instant, + state: Mutex, +} + +impl BuildProgressCallbacks { + fn state(&self) -> MutexGuard<'_, CallbackState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn locked(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { + within(self.lock.bind(py), |_| body()) + } + + fn layout(&self, py: Python<'_>) -> PyResult { + Ok(ProgressLayout { + total: self.facts.total, + name_width: self.facts.name_width, + columns: terminal_columns(py)?, + verbose: self.verbose, + }) + } + + fn emit(&self, py: Python<'_>, text: &str) -> PyResult<()> { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, text)?; + flush(stream) + } + + fn write_spinner_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || { + let (row, frame) = { + let mut state = self.state(); + if !state.line_active { + return Ok(()); + } + let frame: usize = state.frame; + state.frame = (frame + 1) % SPINNER_FRAMES.len(); + ( + SpinnerRow { + position: state.counter + 1, + resource_kind: state.node_kind.clone(), + name: state.node_name.clone(), + message: state.sub_message.clone(), + }, + frame, + ) + }; + self.emit( + py, + &render_build_spinner_line(&row, frame, self.layout(py)?, self.style), + ) + }) + } + + fn set_cursor(&self, py: Python<'_>, hidden: bool) -> PyResult<()> { + if self.state().cursor_hidden == hidden { + return Ok(()); + } + self.locked(py, || { + self.emit(py, if hidden { HIDE_CURSOR } else { SHOW_CURSOR }) + })?; + self.state().cursor_hidden = hidden; + Ok(()) + } + + fn stop_spinner(&self, py: Python<'_>) -> PyResult<()> { + let (event, thread) = { + let mut state = self.state(); + (state.stop_event.take(), state.thread.take()) + }; + if let Some(event) = event { + event.bind(py).call_method0("set")?; + } + if let Some(thread) = thread { + join_spinner(thread.bind(py))?; + } + Ok(()) + } + + /// Stop the spinner and erase its row before a persistent row is written. + fn retire_live_row(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let callbacks: &Self = slf.get(); + callbacks.stop_spinner(py)?; + if !callbacks.is_tty { + return Ok(()); + } + callbacks.locked(py, || { + callbacks.state().line_active = false; + release(callbacks.lines.bind(py), slf.as_any())?; + callbacks.emit(py, CLEAR_LINE) + })?; + callbacks.set_cursor(py, false) + } + + fn write_runtime_diagnostic(&self, py: Python<'_>, message: &str) -> PyResult<()> { + imported(py, "logging", "getLogger")? + .call1((RUNTIME_LOGGER,))? + .call_method1("debug", (message,))?; + if self.debug { + return Ok(()); + } + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("stream", self.stream.bind(py))?; + options.set_item("text", format!("{message}\n"))?; + self.lines + .bind(py) + .call_method("write_persistent", (), Some(&options))?; + Ok(()) + } + + fn write_node_result( + slf: &Bound<'_, Self>, + result: &Bound<'_, PyAny>, + kind: ResultKind, + ) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let callbacks: &Self = slf.get(); + if kind == ResultKind::SqlTest { + let test: TestResultText = test_text(result)?; + if test.outcome != PASS_OUTCOME { + Self::retire_live_row(slf)?; + return callbacks.emit( + py, + &render_failed_test_result(&test, callbacks.layout(py)?, callbacks.style), + ); + } + let mut state = callbacks.state(); + for step in &test.steps { + state + .tests_by_model + .entry(step.model_name.clone()) + .or_default() + .push(test.clone()); + } + return Ok(()); + } + Self::retire_live_row(slf)?; + let counter: usize = { + let mut state = callbacks.state(); + state.counter += 1; + state.counter + }; + let layout: ProgressLayout = callbacks.layout(py)?; + let style: CliStyle = callbacks.style; + let text: String = match kind { + ResultKind::Seed => render_resource_result( + ResourceResult::Node(&seed_text(result)?), + counter, + layout, + style, + ), + ResultKind::Function => render_resource_result( + ResourceResult::Node(&function_text(result)?), + counter, + layout, + style, + ), + ResultKind::Load => render_resource_result( + ResourceResult::Load(&load_text(result)?), + counter, + layout, + style, + ), + ResultKind::Model => { + let model = model_text(result)?; + let facts: &ModelRowFacts = callbacks + .facts + .models + .get(&model.name) + .unwrap_or(&callbacks.facts.unplanned_model); + let tests: Vec = callbacks + .state() + .tests_by_model + .get(&model.name) + .cloned() + .unwrap_or_default(); + render_model_result((&model, facts, &tests), counter, layout, style) + } + _ => return Ok(()), + }; + callbacks.emit(py, &text) + } + + fn write_event(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { + let writer: &Bound<'_, PyAny> = self.event_writer.bind(py); + let write_one = |item: &Bound<'_, PyAny>| -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("result", item)?; + options.set_item("plan", self.plan.bind(py))?; + writer.call_method("write_build_result", (), Some(&options))?; + Ok(()) + }; + write_one(result)?; + if result_kind(result)? == ResultKind::Model { + for audit in result.getattr("audit_results")?.try_iter()? { + write_one(&audit?)?; + } + } + Ok(()) + } +} + +fn scheduler_state(state: &Bound<'_, PyAny>) -> PyResult { + Ok(SchedulerStateText { + running: state.getattr("running")?.extract()?, + ready: state.getattr("ready")?.extract()?, + waiting: state.getattr("waiting")?.extract()?, + limit: state.getattr("limit")?.extract()?, + aborted: state.getattr("aborted")?.extract()?, + }) +} + +fn statement(telemetry: &Bound<'_, PyAny>) -> PyResult { + Ok(StatementText { + query_id: optional(telemetry, "query_id")?, + status: text(telemetry, "status")?, + elapsed_seconds: telemetry.getattr("elapsed_seconds")?.extract()?, + resource_type: text(telemetry, "resource_type")?, + resource_name: text(telemetry, "resource_name")?, + phase: text(telemetry, "phase")?, + }) +} + +#[pymethods] +impl BuildProgressCallbacks { + #[new] + #[pyo3(signature = (**options))] + fn new(py: Python<'_>, options: Option>) -> PyResult { + let arguments = KeywordArguments::read( + py, + "BuildProgressCallbacks", + options, + ( + &["plan", "use_color"], + &["verbose", "debug", "event_output_path"], + ), + )?; + let flag = |name: &str| -> PyResult { + Ok(arguments.passed(name)? && arguments.value(name)?.is_truthy()?) + }; + let plan: Bound<'_, PyAny> = arguments.value("plan")?; + let debug: bool = flag("debug")?; + let sys: Bound<'_, PyAny> = py.import("sys")?.into_any(); + let stdout: Bound<'_, PyAny> = sys.getattr("stdout")?; + let stream: Bound<'_, PyAny> = if debug { + sys.getattr("stderr")? + } else { + stdout.clone() + }; + let path: Bound<'_, PyAny> = if arguments.passed("event_output_path")? { + arguments.value("event_output_path")? + } else { + py.None().into_bound(py) + }; + let writer_options: Bound<'_, PyDict> = PyDict::new(py); + writer_options.set_item("path", path)?; + let event_writer: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "ExecutionEventWriter")?.call((), Some(&writer_options))?; + let facts: BuildPlanFacts = build_plan_facts(&plan)?; + let projector: Option> = projector(py)?; + if let Some(projector) = &projector { + let projector: &Bound<'_, PyAny> = projector.bind(py); + let ordinals: Bound<'_, PyDict> = PyDict::new(py); + for (name, ordinal) in &facts.ordinals { + ordinals.set_item(name, ordinal)?; + } + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("ordinals", ordinals)?; + options.set_item("total", facts.total)?; + projector.call_method("configure_resources", (), Some(&options))?; + for name in &facts.test_names { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("resource_name", name)?; + projector.call_method("expect_resource_enrichment", (), Some(&options))?; + } + } + let lines: Bound<'_, PyAny> = shared_lines(py)?; + Ok(Self { + plan: plan.unbind(), + is_tty: tty_flag(&stdout)? && !debug, + stream: stream.unbind(), + lock: lines.getattr("lock")?.unbind(), + lines: lines.unbind(), + event_writer: event_writer.unbind(), + projector, + facts, + style: CliStyle { + use_color: arguments.value("use_color")?.is_truthy()?, + }, + verbose: flag("verbose")?, + debug, + started: Instant::now(), + state: Mutex::new(CallbackState::default()), + }) + } + + #[getter] + fn elapsed(&self) -> f64 { + self.started.elapsed().as_secs_f64() + } + + #[pyo3(signature = (*, name, resource_kind))] + fn on_node_start( + slf: &Bound<'_, Self>, + name: String, + resource_kind: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let callbacks: &Self = slf.get(); + if let Some(projector) = &callbacks.projector { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("resource_name", name)?; + projector + .bind(py) + .call_method("expect_resource_enrichment", (), Some(&options))?; + return Ok(()); + } + { + let mut state = callbacks.state(); + state.node_name = name; + state.node_kind = resource_kind.getattr("value")?.extract()?; + state.sub_message = String::new(); + } + if !callbacks.is_tty { + return Ok(()); + } + callbacks.set_cursor(py, true)?; + callbacks.locked(py, || { + callbacks.state().line_active = true; + claim( + callbacks.lines.bind(py), + callbacks.stream.bind(py), + slf.as_any(), + )?; + callbacks.write_spinner_line(py) + })?; + callbacks.stop_spinner(py)?; + let (event, thread) = start_spinner(slf.as_any())?; + let mut state = callbacks.state(); + state.stop_event = Some(event.unbind()); + state.thread = Some(thread.unbind()); + Ok(()) + } + + fn on_sub_progress(&self, py: Python<'_>, message: String) -> PyResult<()> { + self.state().sub_message = message; + if self.is_tty { + self.write_spinner_line(py)?; + } + Ok(()) + } + + /// Report one deduplicated concurrent scheduler state transition. + fn on_scheduler_state(&self, py: Python<'_>, state: &Bound<'_, PyAny>) -> PyResult<()> { + let state: SchedulerStateText = scheduler_state(state)?; + { + let mut callback_state = self.state(); + callback_state.scheduler_count += 1; + if callback_state.scheduler_count > SCHEDULER_DIAGNOSTIC_LIMIT { + callback_state.scheduler_omitted += 1; + callback_state.last_scheduler = Some(state); + return Ok(()); + } + } + self.write_runtime_diagnostic(py, &scheduler_diagnostic_message(state)) + } + + /// Report a bounded sample of completed warehouse statement identities. + fn on_statement_complete(&self, py: Python<'_>, telemetry: &Bound<'_, PyAny>) -> PyResult<()> { + let telemetry: StatementText = statement(telemetry)?; + { + let mut state = self.state(); + state.query_count += 1; + if state.query_count > QUERY_DIAGNOSTIC_LIMIT { + state.query_omitted += 1; + state.last_query = Some(telemetry); + return Ok(()); + } + } + self.write_runtime_diagnostic(py, &query_diagnostic_message(&telemetry)) + } + + /// Erase the live spinner row so a persistent line can take its place. + fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || self.emit(py, CLEAR_LINE)) + } + + /// Draw the live spinner row again below persistent output. + fn redraw_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.write_spinner_line(py) + } + + fn on_node_complete(slf: &Bound<'_, Self>, node_result: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let callbacks: &Self = slf.get(); + let kind: ResultKind = result_kind(node_result)?; + let mut result: Bound<'_, PyAny> = node_result.clone(); + if let Some(projector) = &callbacks.projector + && let Some((name, identity)) = resource_identity(node_result, kind)? + { + let Some(duration_ms) = consume_terminal(projector.bind(py), &name, Some(&identity))? + else { + return Ok(()); + }; + if matches!( + kind, + ResultKind::Model | ResultKind::Seed | ResultKind::Function | ResultKind::Load + ) { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("duration_ms", duration_ms.trunc() as i64)?; + result = + imported(py, "dataclasses", "replace")?.call((node_result,), Some(&options))?; + } + } + let written: PyResult<()> = Self::write_node_result(slf, &result, kind); + let evented: PyResult<()> = callbacks.write_event(py, &result); + written.and(evented) + } + + /// Write one result to the shared structured event channel. + fn write_execution_event(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { + self.write_event(py, result) + } + + /// Close the optional structured execution event stream. + fn close(&self, py: Python<'_>) -> PyResult<()> { + let (scheduler, query) = { + let state = self.state(); + ( + state + .last_scheduler + .map(|last| (state.scheduler_omitted, last)), + state + .last_query + .clone() + .map(|last| (state.query_omitted, last)), + ) + }; + for line in runtime_diagnostic_summary( + scheduler, + query.as_ref().map(|(omitted, last)| (*omitted, last)), + ) { + self.write_runtime_diagnostic(py, &line)?; + } + self.event_writer.bind(py).call_method0("close")?; + Ok(()) + } + + fn _spin_until_stopped(&self, py: Python<'_>, stop_event: &Bound<'_, PyAny>) -> PyResult<()> { + while !stop_event + .call_method1("wait", (SPINNER_TICK_SECONDS,))? + .is_truthy()? + { + self.write_spinner_line(py)?; + } + Ok(()) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs index a316c0c55c..2785c8691b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs @@ -1,6 +1,8 @@ pub(crate) mod audit_progress; +pub(crate) mod build_callbacks; pub(crate) mod footer; pub(crate) mod nested_progress; +pub(crate) mod plan_facts; pub(crate) mod registration; pub(crate) mod result_text; pub(crate) mod terminal; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs new file mode 100644 index 0000000000..3c788606b1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs @@ -0,0 +1,120 @@ +//! The plan facts build progress rows need: positions, name width and model row labels. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_core::build_progress::main::progress_name_width::progress_name_width; +use sqlbuild_core::build_progress::models::{ModelRowFacts, NameWidthInputs}; +use sqlbuild_core::plan_display::main::model_execution_annotation::model_execution_annotation; +use sqlbuild_core::plan_display::main::model_resource_type::model_resource_type; +use sqlbuild_core::plan_display::models::ModelLabelFacts; + +use crate::bindings::_helpers::build_progress::result_text::text; +use crate::bindings::_helpers::plan_output::display::optional_facts; + +const SOURCE_TYPE: &str = "source"; +const SQL_TEST_TYPE: &str = "sql_test"; + +/// What a build run's progress rows read from its plan. +pub(crate) struct BuildPlanFacts { + pub(crate) total: usize, + /// Display positions of every progress row, for the progress projector. + pub(crate) ordinals: Vec<(String, usize)>, + pub(crate) test_names: Vec, + pub(crate) name_width: usize, + pub(crate) models: HashMap, + /// The labels of a result whose model is not in the plan. + pub(crate) unplanned_model: ModelRowFacts, +} + +fn row_facts(facts: Option<&ModelLabelFacts>, logical_ddl: Option) -> ModelRowFacts { + ModelRowFacts { + display_type: model_resource_type(facts).to_owned(), + annotation: model_execution_annotation(facts), + logical_ddl, + } +} + +fn names(plan: &Bound<'_, PyAny>, field: &str) -> PyResult> { + plan.getattr(field)? + .try_iter()? + .map(|entry| text(&entry?, "name")) + .collect() +} + +/// Whether the execution-order key is a source with a loader. +fn loaded_source(plan: &Bound<'_, PyAny>, name: &Bound<'_, PyAny>) -> PyResult { + let source: Bound<'_, PyAny> = plan.getattr("source_map")?.call_method1("get", (name,))?; + Ok(!source.is_none() && !source.getattr("loader")?.is_none()) +} + +fn test_tree_children(plan: &Bound<'_, PyAny>) -> PyResult> { + let mut children: Vec = Vec::new(); + for entry in plan.getattr("test_entries")?.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + for step in entry.getattr("chain")?.try_iter()? { + let step: Bound<'_, PyAny> = step?; + if step.getattr("expected_cte_sql")?.is_truthy()? { + children.push(text(&step, "model_name")?); + } + } + for assertion in entry.getattr("assertions")?.try_iter()? { + children.push(format!("assertion {}", text(&assertion?, "name")?)); + } + } + Ok(children) +} + +pub(crate) fn build_plan_facts(plan: &Bound<'_, PyAny>) -> PyResult { + let mut models: HashMap = HashMap::new(); + let mut row_names: Vec = Vec::new(); + for entry in plan.getattr("model_entries")?.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + let name: String = text(&entry, "name")?; + let facts: ModelRowFacts = row_facts( + optional_facts(&entry)?.as_ref(), + Some(text(&entry, "logical_ddl")?), + ); + row_names.push(if facts.annotation.is_empty() { + name.clone() + } else { + format!("{name} ({})", facts.annotation) + }); + models.insert(name, facts); + } + let model_count: usize = models.len(); + let seeds: Vec = names(plan, "seed_entries")?; + let functions: Vec = names(plan, "function_entries")?; + let test_names: Vec = names(plan, "test_entries")?; + let mut ordinals: Vec<(String, usize)> = Vec::new(); + let mut loaded_sources: usize = 0; + for key in plan.getattr("execution_order")?.try_iter()? { + let key: Bound<'_, PyAny> = key?; + let resource_type: String = text(&key, "resource_type")?; + let name: Bound<'_, PyAny> = key.getattr("name")?; + if resource_type == SQL_TEST_TYPE { + continue; + } + if resource_type == SOURCE_TYPE { + if !loaded_source(plan, &name)? { + continue; + } + loaded_sources += 1; + } + ordinals.push((name.str()?.to_string(), ordinals.len() + 1)); + } + row_names.extend(seeds.iter().cloned()); + row_names.extend(functions.iter().cloned()); + row_names.extend(test_names.iter().cloned()); + Ok(BuildPlanFacts { + total: model_count + seeds.len() + functions.len() + loaded_sources, + ordinals, + name_width: progress_name_width(&NameWidthInputs { + names: row_names, + tree_child_names: test_tree_children(plan)?, + }), + test_names, + models, + unplanned_model: row_facts(None, None), + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs index dd7135ce24..42ec0ba298 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs @@ -2,10 +2,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::build_progress::{audit_progress, footer, nested_progress}; +use crate::bindings::_helpers::build_progress::{ + audit_progress, build_callbacks, footer, nested_progress, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { audit_progress::register(module)?; + build_callbacks::register(module)?; footer::register(module)?; nested_progress::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs index 88fd9ad7aa..e7f3325b56 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs @@ -14,7 +14,7 @@ const PYTHON_NODE_MODELS: &str = "sqlbuild.executor.python_nodes.models"; const TESTING_MODELS: &str = "sqlbuild.executor.testing.models"; const AUDITING_MODELS: &str = "sqlbuild.executor.auditing.models"; const CLONE_MODELS: &str = "sqlbuild.executor.clone.models"; -const SQL_TEST_ID_MODULE: &str = "sqlbuild.executor.testing.main.resource_id"; +const SQL_TEST_ID_MODULE: &str = "sqlbuild.executor.testing.main._resource_id"; const AUDIT_ID_MODULE: &str = "sqlbuild.executor.auditing.main.resource_id"; /// The executor result types that result output distinguishes. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 1576b1b915..5c20ac5e8e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -40,7 +40,7 @@ const PROGRESS_MODELS: &str = "sqlbuild.cli.progress.models"; const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; const EXECUTOR_PIPELINE_MODELS: &str = "sqlbuild.executor.pipeline.models"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; -const RESOURCE_ID_MODULE: &str = "sqlbuild.executor.testing.main.resource_id"; +const RESOURCE_ID_MODULE: &str = "sqlbuild.executor.testing.main._resource_id"; const RUNTIME_TARGET_MODULE: &str = "sqlbuild.cli.target_artifacts.main._write_test_runtime_target"; const DECIMAL_KIND: &str = "decimal"; const PASS_OUTCOME: &str = "pass"; diff --git a/python-boundary.toml b/python-boundary.toml index 9c2976c0aa..957d25f2d7 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 61597 +remaining_lines = 59985 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5d5b012e9b..04d0a6a207 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -240,7 +240,7 @@ from sqlbuild.compiler.sql_test_glue.models import ( NativeSqlTestPlanningRequest, ) from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestPlanRow -from sqlbuild.cost.models import CostRunRecord +from sqlbuild.cost.models import CostRunRecord, StatementExecutionTelemetry from sqlbuild.cursor_algebra.models import ( AlignedInterval, DateValue, @@ -249,7 +249,11 @@ from sqlbuild.cursor_algebra.models import ( ) from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult +from sqlbuild.executor.build.models import ( + BuildExecutionResult, + SchedulerState, + SeedExecutionResult, +) from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult from sqlbuild.executor.diff.models import DiffExecutionResult from sqlbuild.executor.janitor.models import JanitorExecutionResult, JanitorPlan @@ -3191,6 +3195,28 @@ class LoadProgressReporter: def clear_transient_line(self) -> None: ... def redraw_transient_line(self) -> None: ... +class BuildProgressCallbacks: + def __init__( + self, + *, + plan: PlanOutput, + use_color: bool, + verbose: bool = False, + debug: bool = False, + event_output_path: Path | None = None, + ) -> None: ... + @property + def elapsed(self) -> float: ... + def on_node_start(self, *, name: str, resource_kind: ExecutionResourceKind) -> None: ... + def on_sub_progress(self, message: str) -> None: ... + def on_scheduler_state(self, state: SchedulerState) -> None: ... + def on_statement_complete(self, telemetry: StatementExecutionTelemetry) -> None: ... + def clear_transient_line(self) -> None: ... + def redraw_transient_line(self) -> None: ... + def on_node_complete(self, node_result: object) -> None: ... + def write_execution_event(self, result: object) -> None: ... + def close(self) -> None: ... + class AuditProgressReporter: def __init__( self, diff --git a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py index edc2843934..8132be96ca 100644 --- a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py +++ b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py @@ -7,15 +7,13 @@ from typing import TextIO from sqlbuild._native import ( + BuildProgressCallbacks, ConnectionProgressReporter, enforce_plan_safety_policies, + format_build_footer, format_execution_header, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands.classes.build_progress_callbacks import ( - BuildProgressCallbacks, - format_build_footer, -) from sqlbuild.cli.commands.constants import ( EXECUTION_LIMIT_DBT_NOTE, MISSING_ORIGIN_DBT_HELP, diff --git a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py deleted file mode 100644 index 48a1ea6a9e..0000000000 --- a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py +++ /dev/null @@ -1,1580 +0,0 @@ -"""Build progress output callbacks and summary formatting.""" - -from __future__ import annotations - -import logging -import sys -import threading -import time -from dataclasses import replace -from pathlib import Path - -from sqlbuild._native import ( - ExecutionEventWriter, - NativeProgressProjector, - current_native_progress_projector, - cursor_bound_display, - format_completion_line, - format_expectation_detail, - format_expectation_name, - format_no_input_rows, - format_parameterized_test_label, - format_status_cell, - format_summary_footer, - format_waiting_on_empty_inputs, - inclusive_cursor_end, - materialization_type_display, - model_execution_annotation, - model_resource_type, -) -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.progress.models import AuditDisplayEntry, ExecutionCounts -from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditOutcome, AuditRunScope -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput -from sqlbuild.compiler.python_nodes.types import PythonNodeStatus -from sqlbuild.cost.models import StatementExecutionTelemetry -from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.executor.build.models import ( - BuildExecutionResult, - FunctionExecutionResult, - SchedulerState, - SeedExecutionResult, -) -from sqlbuild.executor.build.types import BuildStatus, ExecutionStatus -from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult -from sqlbuild.executor.run.models import HookExecutionResult, ModelExecutionResult -from sqlbuild.executor.run.types import HookPhase -from sqlbuild.executor.testing.main.resource_id import sql_test_resource_id -from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult -from sqlbuild.executor.testing.types import SqlTestOutcome -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator -from sqlbuild.presentation.main.coded_error_text import format_coded_error -from sqlbuild.presentation.main.inline_error_lines import format_inline_error_lines -from sqlbuild.presentation.main.terminal_columns import terminal_columns -from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator -from sqlbuild.presentation.main.tree_connector import tree_connector -from sqlbuild.presentation.types import CompletionState -from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry - -_TYPE_WIDTH: int = 10 -_MIN_NAME_WIDTH: int = 20 -_NAME_PADDING: int = 2 -_SUB_INDENT: int = 2 -_TREE_CHILD_NAME_INDENT: int = _SUB_INDENT + 4 -_HOOK_PHASE_WIDTH: int = 10 -_HOOK_TYPE_WIDTH: int = 8 -_HOOK_MIN_LABEL_WIDTH: int = 24 -_HOOK_MAX_LABEL_WIDTH: int = 56 -_SPINNER_TICK_SECONDS: float = 0.1 -_SKIPPED_ROLLUP_CAP: int = 20 -_ROW_COUNT_BILLION: int = 1_000_000_000 -_ROW_COUNT_MILLION: int = 1_000_000 -_ROW_COUNT_THOUSAND: int = 1_000 -_ROW_COUNT_ABBREVIATION_THRESHOLD: int = 10_000 -_ACTIVE_SPINNER_FRAMES: tuple[str, ...] = ( - "⠋", - "⠙", - "⠹", - "⠸", - "⠼", - "⠴", - "⠦", - "⠧", - "⠇", - "⠏", -) -_RUNTIME_DIAGNOSTICS_LOGGER: logging.Logger = logging.getLogger("sqlbuild.executor.runtime") -_SCHEDULER_DIAGNOSTIC_LIMIT: int = 50 -_QUERY_DIAGNOSTIC_LIMIT: int = 25 - - -def _result_resource_name(result: object) -> str | None: - if isinstance(result, ModelExecutionResult): - return result.model_name - if isinstance(result, SeedExecutionResult): - return result.seed_name - if isinstance(result, FunctionExecutionResult): - return result.function_name - if isinstance(result, LoadExecutionResult): - return result.source_name - if isinstance(result, PythonNodeExecutionResult): - return result.node_name - if isinstance(result, SqlTestExecutionResult): - return result.test_name - return None - - -def _result_resource_id(result: object) -> str | None: - if isinstance(result, ModelExecutionResult): - return f"model:{result.model_name}" - if isinstance(result, SeedExecutionResult): - return f"seed:{result.seed_name}" - if isinstance(result, FunctionExecutionResult): - return f"{result.function_kind}:{result.function_name}" - if isinstance(result, LoadExecutionResult): - return f"source:{result.source_name}" - if isinstance(result, PythonNodeExecutionResult): - return f"{result.kind.value}:{result.node_name}" - if isinstance(result, SqlTestExecutionResult): - return sql_test_resource_id( - test_name=result.test_name, - source_path=result.source_path, - block_index=result.block_index, - case_name=result.case_name, - ) - return None - - -def _with_canonical_duration(*, result: object, duration_ms: float) -> object: - if isinstance(result, ModelExecutionResult): - return replace(result, duration_ms=int(duration_ms)) - if isinstance(result, SeedExecutionResult): - return replace(result, duration_ms=int(duration_ms)) - if isinstance(result, FunctionExecutionResult): - return replace(result, duration_ms=int(duration_ms)) - if isinstance(result, LoadExecutionResult): - return replace(result, duration_ms=int(duration_ms)) - return result - - -class BuildProgressCallbacks: - """Encapsulates live build progress output state and callbacks.""" - - def __init__( - self, - *, - plan: PlanOutput, - use_color: bool, - verbose: bool = False, - debug: bool = False, - event_output_path: Path | None = None, - ) -> None: - self._plan: PlanOutput = plan - self._model_entry_map: dict[str, ModelPlanEntry] = { - entry.name: entry for entry in plan.model_entries - } - self._test_results_by_model: dict[str, list[SqlTestExecutionResult]] = {} - self._total: int = ( - len(plan.model_entries) - + len(plan.seed_entries) - + len(plan.function_entries) - + sum( - 1 - for key in plan.execution_order - if key.resource_type == CompiledResourceType.SOURCE - and plan.source_map.get(key.name) is not None - and plan.source_map[key.name].loader is not None - ) - ) - self._counter: int = 0 - self._use_color: bool = use_color - self._style: CliStyle = CliStyle(use_color=self._use_color) - self._verbose: bool = verbose - self._debug: bool = debug - self._is_tty: bool = hasattr(sys.stdout, "isatty") and sys.stdout.isatty() and not debug - self._stream = sys.stderr if debug else sys.stdout - self._lines: TransientLineCoordinator = shared_transient_line_coordinator() - self._start_time: float = time.monotonic() - self._current_node_name: str = "" - self._current_node_type: ExecutionResourceKind = ExecutionResourceKind.TABLE - self._current_sub_message: str = "" - self._spinner_frame_index: int = 0 - self._spinner_line_active: bool = False - self._runtime_diagnostic_lock: threading.Lock = threading.Lock() - self._spinner_stop_event: threading.Event | None = None - self._spinner_thread: threading.Thread | None = None - self._cursor_hidden: bool = False - self._event_writer: ExecutionEventWriter = ExecutionEventWriter(path=event_output_path) - self._scheduler_diagnostic_count: int = 0 - self._scheduler_diagnostic_omitted: int = 0 - self._last_omitted_scheduler_state: SchedulerState | None = None - self._query_diagnostic_count: int = 0 - self._query_diagnostic_omitted: int = 0 - self._last_omitted_query: StatementExecutionTelemetry | None = None - self._projector: NativeProgressProjector | None = current_native_progress_projector() - if self._projector is not None: - display_names: list[str] = [] - for key in plan.execution_order: - if key.resource_type == CompiledResourceType.SQL_TEST: - continue - if key.resource_type == CompiledResourceType.SOURCE: - source: SourceEntry | None = plan.source_map.get(key.name) - if source is None or source.loader is None: - continue - display_names.append(key.name) - self._projector.configure_resources( - ordinals={name: index for index, name in enumerate(display_names, start=1)}, - total=self._total, - ) - for test_entry in plan.test_entries: - self._projector.expect_resource_enrichment(resource_name=test_entry.name) - - ctr_width: int = len(str(self._total)) * 2 + 1 - self._prefix_width: int = 2 + ctr_width + 2 - - max_name_len: int = 0 - entry: ModelPlanEntry - for entry in plan.model_entries: - annotation: str = model_execution_annotation(entry) - display_name: str = entry.name - if annotation: - display_name = f"{entry.name} ({annotation})" - max_name_len = max(max_name_len, len(display_name)) - seed_entry: object - for seed_entry in plan.seed_entries: - max_name_len = max(max_name_len, len(getattr(seed_entry, "name", str(seed_entry)))) - function_entry: object - for function_entry in plan.function_entries: - max_name_len = max( - max_name_len, len(getattr(function_entry, "name", str(function_entry))) - ) - test_entry: object - for test_entry in plan.test_entries: - max_name_len = max(max_name_len, len(getattr(test_entry, "name", str(test_entry)))) - chain_step: object - for chain_step in getattr(test_entry, "chain", ()): - if getattr(chain_step, "expected_cte_sql", None): - max_name_len = max( - max_name_len, - _tree_child_name_length( - format_expectation_name(str(getattr(chain_step, "model_name", ""))) - ), - ) - assertion: object - for assertion in getattr(test_entry, "assertions", ()): - max_name_len = max( - max_name_len, - _tree_child_name_length( - format_expectation_name(f"assertion {getattr(assertion, 'name', '')}") - ), - ) - self._name_width: int = max(max_name_len + _NAME_PADDING, _MIN_NAME_WIDTH) - - @property - def elapsed(self) -> float: - return time.monotonic() - self._start_time - - def _write_sql_block(self, sql: str) -> None: - """Write a SQL block with minimal indent and dim styling.""" - - self._stream.write("\n") - sql_line: str - for sql_line in _format_display_sql(sql).split("\n"): - styled: str = self._style.muted(f" {sql_line}") - self._stream.write(f"{styled}\n") - self._stream.write("\n") - - def _write_log_block(self, message: str) -> None: - """Write a log message with indent and muted styling.""" - - lines: list[str] = message.splitlines() or [""] - prefix: str = self._style.log_label(" log ") - first_content: str = self._style.muted(lines[0]) - styled_first: str = f"{prefix}{first_content}" - self._stream.write(f"\n{styled_first}\n") - line: str - for line in lines[1:]: - continuation: str = f" {line}" - styled_continuation: str = self._style.muted(continuation) - self._stream.write(f"{styled_continuation}\n") - - def on_node_start(self, *, name: str, resource_kind: ExecutionResourceKind) -> None: - if self._projector is not None: - self._projector.expect_resource_enrichment(resource_name=name) - return - self._current_node_name = name - self._current_node_type = resource_kind - self._current_sub_message = "" - if self._is_tty: - self._hide_cursor() - with self._lines.lock: - self._spinner_line_active = True - self._lines.claim(stream=self._stream, owner=self) - self._write_spinner_line() - self._start_spinner_loop() - - def on_sub_progress(self, message: str) -> None: - self._current_sub_message = message - if self._is_tty: - self._write_spinner_line() - - def on_scheduler_state(self, state: SchedulerState) -> None: - """Report one deduplicated concurrent scheduler state transition.""" - - with self._runtime_diagnostic_lock: - self._scheduler_diagnostic_count += 1 - if self._scheduler_diagnostic_count > _SCHEDULER_DIAGNOSTIC_LIMIT: - self._scheduler_diagnostic_omitted += 1 - self._last_omitted_scheduler_state = state - return - self._write_runtime_diagnostic(_scheduler_diagnostic_message(state)) - - def on_statement_complete(self, telemetry: StatementExecutionTelemetry) -> None: - """Report a bounded sample of completed Snowflake statement identities.""" - - with self._runtime_diagnostic_lock: - self._query_diagnostic_count += 1 - if self._query_diagnostic_count > _QUERY_DIAGNOSTIC_LIMIT: - self._query_diagnostic_omitted += 1 - self._last_omitted_query = telemetry - return - self._write_runtime_diagnostic(_query_diagnostic_message(telemetry)) - - def _write_runtime_diagnostic(self, message: str) -> None: - _RUNTIME_DIAGNOSTICS_LOGGER.debug(message) - if self._debug: - return - self._lines.write_persistent(stream=self._stream, text=f"{message}\n") - - def _write_runtime_diagnostic_summaries(self) -> None: - with self._runtime_diagnostic_lock: - scheduler_omitted: int = self._scheduler_diagnostic_omitted - last_scheduler_state: SchedulerState | None = self._last_omitted_scheduler_state - query_omitted: int = self._query_diagnostic_omitted - last_query: StatementExecutionTelemetry | None = self._last_omitted_query - if scheduler_omitted and last_scheduler_state is not None: - self._write_runtime_diagnostic( - f"Scheduler diagnostics {scheduler_omitted} transitions omitted " - f"after {_SCHEDULER_DIAGNOSTIC_LIMIT}; final state: " - f"{_scheduler_diagnostic_message(last_scheduler_state)}" - ) - if query_omitted and last_query is not None: - self._write_runtime_diagnostic( - f"Query diagnostics {query_omitted} statements omitted after " - f"{_QUERY_DIAGNOSTIC_LIMIT}; final query: " - f"{_query_diagnostic_message(last_query)}" - ) - - def clear_transient_line(self) -> None: - """Erase the live spinner row so a persistent line can take its place.""" - - with self._lines.lock: - self._stream.write("\r\033[K") - self._stream.flush() - - def redraw_transient_line(self) -> None: - """Draw the live spinner row again below persistent output.""" - - self._write_spinner_line() - - def _write_spinner_line(self) -> None: - with self._lines.lock: - if self._spinner_line_active: - self._draw_spinner_line() - - def _draw_spinner_line(self) -> None: - ctr: str = f"{self._counter + 1}/{self._total}".rjust(len(str(self._total)) * 2 + 1) - display_type: str = materialization_type_display(self._current_node_type) - status: str = self._style.status(status=_ACTIVE_SPINNER_FRAMES[self._spinner_frame_index]) - self._spinner_frame_index = (self._spinner_frame_index + 1) % len(_ACTIVE_SPINNER_FRAMES) - name_display: str = _truncate_name(name=self._current_node_name, width=self._name_width) - if self._current_sub_message: - name_display = f"{name_display} {self._current_sub_message}" - nw: int = self._name_width - plain_part: str = f" {ctr} {display_type:<{_TYPE_WIDTH}}{name_display:<{nw}} " - max_plain_width: int = max(terminal_columns() - 2, _MIN_NAME_WIDTH) - if len(plain_part) > max_plain_width: - plain_part = f"{plain_part[: max_plain_width - 4]}... " - self._stream.write(f"\r\033[K{plain_part}{status}") - self._stream.flush() - - def _start_spinner_loop(self) -> None: - self._stop_spinner_loop() - stop_event: threading.Event = threading.Event() - self._spinner_stop_event = stop_event - spinner_thread: threading.Thread = threading.Thread( - target=self._spin_until_stopped, - args=(stop_event,), - daemon=True, - ) - self._spinner_thread = spinner_thread - spinner_thread.start() - - def _stop_spinner_loop(self) -> None: - if self._spinner_stop_event is not None: - self._spinner_stop_event.set() - if self._spinner_thread is not None and self._spinner_thread.is_alive(): - self._spinner_thread.join(timeout=0.2) - self._spinner_stop_event = None - self._spinner_thread = None - - def _spin_until_stopped(self, stop_event: threading.Event) -> None: - while not stop_event.wait(_SPINNER_TICK_SECONDS): - self._write_spinner_line() - - def _hide_cursor(self) -> None: - if self._cursor_hidden: - return - with self._lines.lock: - self._stream.write("\033[?25l") - self._stream.flush() - self._cursor_hidden = True - - def _show_cursor(self) -> None: - if not self._cursor_hidden: - return - with self._lines.lock: - self._stream.write("\033[?25h") - self._stream.flush() - self._cursor_hidden = False - - def on_node_complete(self, node_result: object) -> None: - if self._projector is not None: - resource_name: str | None = _result_resource_name(node_result) - if resource_name is not None: - duration_ms: float | None = self._projector.consume_resource_terminal( - resource_name=resource_name, - resource_id=_result_resource_id(node_result), - ) - if duration_ms is None: - return - node_result = _with_canonical_duration(result=node_result, duration_ms=duration_ms) - try: - self._write_node_result(node_result) - finally: - self._write_execution_event(node_result) - - def _write_node_result(self, node_result: object) -> None: - if isinstance(node_result, SqlTestExecutionResult): - if node_result.outcome != SqlTestOutcome.PASS: - self._write_failed_test_result(test_result=node_result) - return - step: object - for step in node_result.step_results: - if hasattr(step, "model_name"): - self._test_results_by_model.setdefault(step.model_name, []).append(node_result) - return - - self._stop_spinner_loop() - if self._is_tty: - with self._lines.lock: - self._spinner_line_active = False - self._lines.release(owner=self) - self._stream.write("\r\033[K") - self._stream.flush() - self._show_cursor() - - self._counter += 1 - ctr: str = f"{self._counter}/{self._total}".rjust(len(str(self._total)) * 2 + 1) - - if isinstance(node_result, SeedExecutionResult): - status: str = _execution_status_display(node_result.status) - duration: str = _format_duration(node_result.duration_ms) - seed_name: str = _truncate_name(name=node_result.seed_name, width=self._name_width) - self._write_top_level_result_line( - ctr=ctr, - resource_type="seed", - name=seed_name, - status=status, - duration=duration, - ) - if node_result.status == ExecutionStatus.FAILED and node_result.error_message: - self._write_error_detail( - error_code=node_result.error_code, - error_message=node_result.error_message, - error_help=node_result.error_help, - ) - self._stream.flush() - return - - if isinstance(node_result, FunctionExecutionResult): - status: str = _execution_status_display(node_result.status) - duration: str = _format_duration(node_result.duration_ms) - function_name: str = _truncate_name( - name=node_result.function_name, width=self._name_width - ) - self._write_top_level_result_line( - ctr=ctr, - resource_type=node_result.function_kind, - name=function_name, - status=status, - duration=duration, - ) - if node_result.status == ExecutionStatus.FAILED and node_result.error_message: - self._write_error_detail( - error_code=node_result.error_code, - error_message=node_result.error_message, - error_help=node_result.error_help, - ) - self._stream.flush() - return - - if isinstance(node_result, LoadExecutionResult): - status = _execution_status_display(node_result.status) - duration = _format_duration(node_result.duration_ms) - source_name: str = _truncate_name(name=node_result.source_name, width=self._name_width) - detail: str = _load_result_detail(node_result) - self._write_top_level_result_line( - ctr=ctr, - resource_type=node_result.resource_kind.value, - name=source_name, - status=status, - duration=duration, - detail=detail, - ) - event: LifeCycleEvent - for event in node_result.lifecycle_events: - if event.kind == LifeCycleEventKind.LOG: - self._write_log_block(event.content) - elif self._verbose and event.kind == LifeCycleEventKind.SQL: - self._write_sql_block(event.content) - if node_result.status == ExecutionStatus.FAILED and node_result.error_message: - self._write_error_detail( - error_code=None, - error_message=node_result.error_message, - error_help=None, - ) - self._stream.flush() - return - - if isinstance(node_result, ModelExecutionResult): - self._write_model_result(ctr=ctr, model_result=node_result) - - def _write_execution_event(self, node_result: object) -> None: - self._event_writer.write_build_result( - result=node_result, - plan=self._plan, - ) - if isinstance(node_result, ModelExecutionResult): - for audit_result in node_result.audit_results: - self._event_writer.write_build_result( - result=audit_result, - plan=self._plan, - ) - - def write_execution_event(self, result: object) -> None: - """Write one result to the shared structured event channel.""" - - self._write_execution_event(result) - - def close(self) -> None: - """Close the optional structured execution event stream.""" - - self._write_runtime_diagnostic_summaries() - self._event_writer.close() - - def _write_model_result(self, *, ctr: str, model_result: ModelExecutionResult) -> None: - plan_entry: ModelPlanEntry | None = self._model_entry_map.get(model_result.model_name) - display_type: str = model_resource_type(plan_entry) - annotation: str = model_execution_annotation(plan_entry) - name_display: str = model_result.model_name - if annotation: - name_display = f"{model_result.model_name} ({annotation})" - name_display = _truncate_name(name=name_display, width=self._name_width) - - status: str = _execution_status_display(model_result.status) - duration: str = _format_duration(model_result.duration_ms) - detail: str = "" - if model_result.status == ExecutionStatus.FAILED and model_result.failed_phase is not None: - detail = f" {model_result.failed_phase}" - elif model_result.status == ExecutionStatus.SKIPPED: - duration = "" - detail = _model_skip_detail(model_result) - - self._write_top_level_result_line( - ctr=ctr, - resource_type=display_type, - name=name_display, - status=status, - duration=duration, - detail=detail, - ) - if self._verbose: - event: LifeCycleEvent - for event in _resolve_verbose_events( - model_result=model_result, - plan_entry=plan_entry, - ): - if event.kind == LifeCycleEventKind.SQL: - self._write_sql_block(event.content) - elif event.kind == LifeCycleEventKind.LOG: - self._write_log_block(event.content) - - batch_line: str | None = _format_streaming_batch_summary( - model_result=model_result - ) or _format_empty_cursor_inputs(model_result=model_result) - if batch_line is not None: - self._stream.write(f"{self._style.muted(batch_line)}\n") - self._stream.flush() - if model_result.waiting_on_empty_inputs and model_result.empty_cursor_inputs: - waiting_line: str = ( - f" {format_waiting_on_empty_inputs(model_result.empty_cursor_inputs)}" - ) - self._stream.write(f"{self._style.warning(waiting_line)}\n") - self._stream.flush() - - sub_pad: str = " " * (self._prefix_width + _SUB_INDENT) - sub_nw: int = self._name_width - _SUB_INDENT - hook_label_width: int = _hook_label_width(model_result.hook_results) - - self._write_hook_results( - hook_results=model_result.hook_results, - phase=HookPhase.PRE_HOOKS, - sub_pad=sub_pad, - label_width=hook_label_width, - ) - - test_result: SqlTestExecutionResult - for test_result in self._test_results_by_model.get(model_result.model_name, []): - test_status: str = self._style.status(status=_test_outcome_display(test_result.outcome)) - test_name: str = format_parameterized_test_label( - name=test_result.test_name, - source_path=test_result.source_path, - parameter_schema=test_result.parameter_schema, - parameter_values=test_result.parameter_values, - ) - self._stream.write( - f"{sub_pad}{self._style.muted(f'{"test":<{_TYPE_WIDTH}}')}" - f"{test_name:<{sub_nw}} {test_status}\n" - ) - expectation_pad: str = f"{sub_pad} " - expectation_type_width: int = _TYPE_WIDTH - 2 - step_result: StepResult - for step_result in test_result.step_results: - expectation_status: str = self._style.status( - status=_test_outcome_display(step_result.outcome) - ) - expectation_name: str = format_expectation_name(step_result.model_name) - expectation_detail: str = format_expectation_detail(step_result) - self._stream.write( - f"{expectation_pad}{self._style.muted(f'{"expect":<{expectation_type_width}}')}" - f"{expectation_name:<{sub_nw}} {expectation_status}{expectation_detail}\n" - ) - - display_audits: list[AuditDisplayEntry] = _aggregate_audit_results( - model_result.audit_results - ) - - entry: AuditDisplayEntry - for entry in display_audits: - audit_status: str = self._style.status(status=_audit_outcome_display(entry.outcome)) - audit_name: str = _truncate_name(name=entry.display_name, width=sub_nw) - audit_detail: str = "" - if entry.reused: - audit_detail = " proof reused" - if entry.outcome != AuditOutcome.PASS and entry.total_row_count > 0: - row_label: str = "row" if entry.total_row_count == 1 else "rows" - audit_detail = f" {entry.total_row_count} {row_label}" - if entry.batch_total > 1: - audit_detail = f" {entry.batch_pass}/{entry.batch_total}" + audit_detail - audit_line: str = ( - f"{sub_pad}{self._style.muted(f'{entry.label:<{_TYPE_WIDTH}}')}" - f"{audit_name:<{sub_nw}} {audit_status}{audit_detail}\n" - ) - self._stream.write(audit_line) - - if self._verbose and entry.executed_sql is not None: - self._write_sql_block(entry.executed_sql) - - self._write_hook_results( - hook_results=model_result.hook_results, - phase=HookPhase.POST_HOOKS, - sub_pad=sub_pad, - label_width=hook_label_width, - ) - - if model_result.status == ExecutionStatus.FAILED and model_result.error_message: - self._write_error_detail( - error_code=model_result.error_code, - error_message=model_result.error_message, - error_help=model_result.error_help, - ) - - self._stream.flush() - - def _write_failed_test_result(self, *, test_result: SqlTestExecutionResult) -> None: - """Write a failing test row inline since its host model rows will not run.""" - - self._stop_spinner_loop() - if self._is_tty: - with self._lines.lock: - self._spinner_line_active = False - self._lines.release(owner=self) - self._stream.write("\r\033[K") - self._stream.flush() - self._show_cursor() - blank_ctr: str = " " * (len(str(self._total)) * 2 + 1) - test_status: str = self._style.status(status=_test_outcome_display(test_result.outcome)) - test_name: str = _truncate_name( - name=format_parameterized_test_label( - name=test_result.test_name, - source_path=test_result.source_path, - parameter_schema=test_result.parameter_schema, - parameter_values=test_result.parameter_values, - ), - width=self._name_width, - ) - self._stream.write( - f" {blank_ctr} {self._style.muted(f'{"test":<{_TYPE_WIDTH}}')}" - f"{test_name:<{self._name_width}} {test_status}\n" - ) - expectation_pad: str = " " * (self._prefix_width + _SUB_INDENT + 2) - expectation_type_width: int = _TYPE_WIDTH - 2 - tree_child_nw: int = self._name_width - _TREE_CHILD_NAME_INDENT - step_result: StepResult - for index, step_result in enumerate(test_result.step_results): - expectation_status: str = self._style.status( - status=_test_outcome_display(step_result.outcome) - ) - expectation_name: str = format_expectation_name(step_result.model_name) - expectation_detail: str = format_expectation_detail(step_result) - last: bool = ( - index == len(test_result.step_results) - 1 and not test_result.error_message - ) - connector: str = tree_connector(style=self._style, last=last) - self._stream.write( - f"{expectation_pad}{connector} " - f"{self._style.muted(f'{"expect":<{expectation_type_width}}')}" - f"{expectation_name:<{tree_child_nw}} {expectation_status}{expectation_detail}\n" - ) - if test_result.error_message: - self._write_error_detail( - error_code=test_result.error_code, - error_message=test_result.error_message, - error_help=test_result.error_help, - tree_child=True, - ) - self._stream.flush() - - def _write_hook_results( - self, - *, - hook_results: tuple[HookExecutionResult, ...], - phase: HookPhase, - sub_pad: str, - label_width: int, - ) -> None: - hook_result: HookExecutionResult - for hook_result in hook_results: - if hook_result.phase != phase: - continue - hook_status: str = self._style.status( - status=_execution_status_display(hook_result.status) - ) - hook_name: str = _truncate_name(name=hook_result.label, width=label_width) - label: str = "pre_hook" if phase == HookPhase.PRE_HOOKS else "post_hook" - detail: str = _hook_skip_detail(hook_result) - self._stream.write( - f"{sub_pad}{self._style.muted(f'{label:<{_HOOK_PHASE_WIDTH}}')}" - f"{self._style.muted(f'{hook_result.hook_type:<{_HOOK_TYPE_WIDTH}}')}" - f"{hook_name:<{label_width}} {hook_status}{detail}\n" - ) - - def _write_top_level_result_line( - self, - *, - ctr: str, - resource_type: str, - name: str, - status: str, - duration: str, - detail: str = "", - ) -> None: - nw: int = self._name_width - status_cell: str = format_status_cell(style=self._style, status=status) - rendered_duration: str = self._style.muted(duration) if duration else "" - rendered_type: str = self._style.muted(f"{resource_type:<{_TYPE_WIDTH}}") - self._stream.write( - f" {ctr} {rendered_type}{name:<{nw}} {status_cell} {rendered_duration}{detail}\n" - ) - - def _write_error_detail( - self, - *, - error_code: str | None, - error_message: str, - error_help: str | None = None, - tree_child: bool = False, - ) -> None: - child_offset: int = _SUB_INDENT + 2 if tree_child else 0 - pad: str = " " * (self._prefix_width + child_offset) - connector: str = tree_connector(style=self._style, last=True) if tree_child else "" - connector_suffix: str = f"{connector} " if tree_child else "" - label_width: int = _TYPE_WIDTH - 3 if tree_child else _TYPE_WIDTH - 1 - label_padding: str = " " * max(0, label_width - len("error")) - label: str = f"{self._style.error_muted('error')}{label_padding}" - plain_prefix_width: int = len(pad) + (4 if tree_child else 0) + label_width + 1 - content_width: int = max(terminal_columns() - plain_prefix_width, 1) - lines: list[str] = format_inline_error_lines( - error_code=error_code, - error_message=error_message, - error_help=error_help, - content_width=content_width, - style=self._style, - ) - continuation_pad: str = " " * plain_prefix_width - for line_index, line in enumerate(lines): - if line_index == 0: - self._stream.write(f"{pad}{connector_suffix}{label} {line}\n") - else: - self._stream.write(f"{continuation_pad}{line}\n") - - -def _build_audit_results(result: BuildExecutionResult) -> tuple[AuditExecutionResult, ...]: - audits: list[AuditExecutionResult] = [] - model_result: ModelExecutionResult - for model_result in result.model_results: - audits.extend(model_result.audit_results) - return (*audits, *result.source_audit_results, *result.end_audit_results) - - -def _count_build_footer_results( - *, - result: BuildExecutionResult, - python_node_results: tuple[PythonNodeExecutionResult, ...], -) -> ExecutionCounts: - pass_count: int = 0 - warn_count: int = 0 - fail_count: int = 0 - skip_count: int = 0 - - model_result: ModelExecutionResult - for model_result in result.model_results: - if model_result.status == ExecutionStatus.SUCCESS and model_result.warning_messages: - warn_count += 1 - elif model_result.status == ExecutionStatus.SUCCESS: - pass_count += 1 - elif model_result.status == ExecutionStatus.FAILED: - fail_count += 1 - elif model_result.status == ExecutionStatus.SKIPPED: - skip_count += 1 - audit_r: AuditExecutionResult - for audit_r in _build_audit_results(result): - if audit_r.outcome == AuditOutcome.PASS: - pass_count += 1 - elif audit_r.outcome == AuditOutcome.WARN: - warn_count += 1 - elif audit_r.outcome == AuditOutcome.ERROR: - fail_count += 1 - - seed_result: SeedExecutionResult - for seed_result in result.seed_results: - if seed_result.status == ExecutionStatus.SUCCESS: - pass_count += 1 - elif seed_result.status == ExecutionStatus.FAILED: - fail_count += 1 - elif seed_result.status == ExecutionStatus.SKIPPED: - skip_count += 1 - - function_result: FunctionExecutionResult - for function_result in result.function_results: - if function_result.status == ExecutionStatus.SUCCESS: - pass_count += 1 - elif function_result.status == ExecutionStatus.FAILED: - fail_count += 1 - elif function_result.status == ExecutionStatus.SKIPPED: - skip_count += 1 - warn_count += len(function_result.warning_messages) - - load_result: LoadExecutionResult - for load_result in result.load_results: - if load_result.status == ExecutionStatus.SUCCESS and load_result.warning_messages: - warn_count += 1 - elif load_result.status == ExecutionStatus.SUCCESS: - pass_count += 1 - elif load_result.status == ExecutionStatus.FAILED: - fail_count += 1 - elif load_result.status == ExecutionStatus.SKIPPED: - skip_count += 1 - - test_r: SqlTestExecutionResult - for test_r in result.test_results: - if test_r.outcome == SqlTestOutcome.PASS: - pass_count += 1 - else: - fail_count += 1 - - python_result: PythonNodeExecutionResult - for python_result in python_node_results: - if python_result.status == PythonNodeStatus.SUCCESS and python_result.warning_messages: - warn_count += 1 - elif python_result.status == PythonNodeStatus.SUCCESS: - pass_count += 1 - elif python_result.status == PythonNodeStatus.FAILED: - fail_count += 1 - elif python_result.status == PythonNodeStatus.SKIPPED: - skip_count += 1 - - return ExecutionCounts( - pass_count=pass_count, - warn_count=warn_count, - fail_count=fail_count, - skip_count=skip_count, - ) - - -def format_build_footer( - *, - result: BuildExecutionResult, - elapsed: float, - use_color: bool, - python_node_results: tuple[PythonNodeExecutionResult, ...] = (), -) -> str: - lines: list[str] = [] - style: CliStyle = CliStyle(use_color=use_color) - python_fail_count: int = sum( - 1 - for python_result in python_node_results - if python_result.status == PythonNodeStatus.FAILED - ) - - counts: ExecutionCounts = _count_build_footer_results( - result=result, python_node_results=python_node_results - ) - insufficient_count: int = result.insufficient_count - elapsed_str: str = f"{elapsed:.2f}s" - counts_summary: str = format_summary_footer( - counts=( - ("PASS", counts.pass_count), - ("WARN", counts.warn_count), - ("FAIL", counts.fail_count), - ("INSUFFICIENT", insufficient_count), - ("SKIP", counts.skip_count), - ("TOTAL", counts.total_count + insufficient_count), - ), - use_color=style.use_color, - elapsed=elapsed_str, - ) - if result.status == BuildStatus.FAILED or python_fail_count: - state: CompletionState = CompletionState.FAIL - label: str = "Completed with errors" - elif result.warning_count > 0 or counts.warn_count > 0: - state = CompletionState.WARN - label = "Completed with warnings" - else: - state = CompletionState.OK - label = "Completed successfully" - warning_lines: list[str] = _format_warning_details( - result=result, python_node_results=python_node_results, style=style - ) - if warning_lines: - lines.extend(warning_lines) - lines.append( - format_completion_line(style=style, state=state, label=label, summary=counts_summary) - ) - - failure_lines: list[str] = _format_failure_details(result=result, style=style) - failure_lines.extend(_format_python_failure_details(results=python_node_results, style=style)) - if failure_lines: - lines.extend(failure_lines) - - skipped_lines: list[str] = _format_skipped_rollup( - result=result, python_node_results=python_node_results, style=style - ) - if skipped_lines: - lines.extend(skipped_lines) - - return "\n".join(lines) - - -def _format_skipped_rollup( - *, - result: BuildExecutionResult, - python_node_results: tuple[PythonNodeExecutionResult, ...], - style: CliStyle, -) -> list[str]: - """Render a dim rollup of nodes skipped because upstream work failed.""" - - skipped_names: list[str] = [ - model_result.model_name - for model_result in result.model_results - if model_result.status == ExecutionStatus.SKIPPED - ] - skipped_names.extend( - seed_result.seed_name - for seed_result in result.seed_results - if seed_result.status == ExecutionStatus.SKIPPED - ) - skipped_names.extend( - function_result.function_name - for function_result in result.function_results - if function_result.status == ExecutionStatus.SKIPPED - ) - skipped_names.extend( - load_result.source_name - for load_result in result.load_results - if load_result.status == ExecutionStatus.SKIPPED - ) - skipped_names.extend( - python_result.node_name - for python_result in python_node_results - if python_result.status == PythonNodeStatus.SKIPPED - ) - if not skipped_names: - return [] - visible_names: list[str] = skipped_names[:_SKIPPED_ROLLUP_CAP] - remaining_count: int = len(skipped_names) - len(visible_names) - rendered: str = ", ".join(visible_names) - if remaining_count > 0: - rendered = f"{rendered}, ... (+{remaining_count} more)" - return [ - "", - style.muted(f"Skipped ({len(skipped_names)}): {rendered}"), - ] - - -def _format_python_failure_details( - *, results: tuple[PythonNodeExecutionResult, ...], style: CliStyle -) -> list[str]: - lines: list[str] = [] - failed_results: tuple[PythonNodeExecutionResult, ...] = tuple( - result for result in results if result.status == PythonNodeStatus.FAILED - ) - if not failed_results: - return lines - lines.append("") - lines.append(style.error_strong("Python node failures:")) - lines.append("") - result: PythonNodeExecutionResult - for result in failed_results: - message: str = result.error_message or "Python node failed" - lines.append(f" {result.kind.value:<10}{result.node_name:<50} {style.error(message)}") - return lines - - -def _format_load_failure_details( - *, results: tuple[LoadExecutionResult, ...], style: CliStyle -) -> list[str]: - lines: list[str] = [] - load_result: LoadExecutionResult - for load_result in results: - if load_result.status != ExecutionStatus.FAILED: - continue - if not lines: - lines.append("") - lines.append(style.error_strong("Failures:")) - lines.append("") - lines.append(f" {load_result.source_name} ({load_result.resource_kind.value})") - if load_result.error_message is not None: - lines.extend( - _format_failure_error_block( - error_code=None, - error_message=load_result.error_message, - error_help=None, - style=style, - ) - ) - lines.append("") - return lines - - -def _format_failure_details(*, result: BuildExecutionResult, style: CliStyle) -> list[str]: - lines: list[str] = _format_load_failure_details(results=result.load_results, style=style) - has_failures: bool = bool(lines) - - seed_result: SeedExecutionResult - for seed_result in result.seed_results: - if seed_result.status != ExecutionStatus.FAILED: - continue - if not has_failures: - lines.append("") - lines.append(style.error_strong("Failures:")) - lines.append("") - has_failures = True - lines.append(f" {seed_result.seed_name} (seed)") - if seed_result.error_message is not None: - lines.extend( - _format_failure_error_block( - error_code=seed_result.error_code, - error_message=seed_result.error_message, - error_help=seed_result.error_help, - style=style, - ) - ) - lines.append("") - - model_result: ModelExecutionResult - for model_result in result.model_results: - if model_result.status != ExecutionStatus.FAILED: - continue - if not has_failures: - lines.append("") - lines.append(style.error_strong("Failures:")) - lines.append("") - has_failures = True - phase_str: str = f" ({model_result.failed_phase})" if model_result.failed_phase else "" - lines.append(f" {model_result.model_name}{phase_str}") - if model_result.error_message is not None: - lines.extend( - _format_failure_error_block( - error_code=model_result.error_code, - error_message=model_result.error_message, - error_help=model_result.error_help, - style=style, - ) - ) - if model_result.staging_relation is not None: - lines.append(f" {_inspection_relation_message(model_result.staging_relation)}") - lines.append("") - - function_result: FunctionExecutionResult - for function_result in result.function_results: - if function_result.status != ExecutionStatus.FAILED: - continue - if not has_failures: - lines.append("") - lines.append(style.error_strong("Failures:")) - lines.append("") - has_failures = True - lines.append(f" {function_result.function_name} ({function_result.function_kind})") - if function_result.error_message is not None: - lines.extend( - _format_failure_error_block( - error_code=function_result.error_code, - error_message=function_result.error_message, - error_help=function_result.error_help, - style=style, - ) - ) - lines.append("") - - test_r: SqlTestExecutionResult - for test_r in result.test_results: - if test_r.outcome == SqlTestOutcome.PASS: - continue - if not has_failures: - lines.append("") - lines.append(style.error_strong("Failures:")) - lines.append("") - has_failures = True - test_label: str = format_parameterized_test_label( - name=test_r.test_name, - source_path=test_r.source_path, - parameter_schema=test_r.parameter_schema, - parameter_values=test_r.parameter_values, - ) - lines.append(f" {test_label} (test)") - if test_r.error_message is not None: - lines.extend( - _format_failure_error_block( - error_code=test_r.error_code, - error_message=test_r.error_message, - error_help=test_r.error_help, - style=style, - ) - ) - lines.append("") - - lines.extend(_format_audit_error_details(result=result, style=style, has_failures=has_failures)) - return lines - - -def _format_audit_error_details( - *, result: BuildExecutionResult, style: CliStyle, has_failures: bool -) -> list[str]: - lines: list[str] = [] - audit_result: AuditExecutionResult - for audit_result in (*result.source_audit_results, *result.end_audit_results): - if audit_result.execution_error is None: - continue - if not has_failures: - lines.append("") - lines.append(style.error_strong("Failures:")) - lines.append("") - has_failures = True - target: str = ( - f" on {audit_result.attached_target_name}" - if audit_result.attached_target_name is not None - else "" - ) - lines.append(f" {audit_result.audit_name}{target} (audit)") - lines.extend( - _format_failure_error_block( - error_code=None, - error_message=audit_result.execution_error, - error_help=None, - style=style, - ) - ) - lines.append("") - - return lines - - -def _format_warning_details( - *, - result: BuildExecutionResult, - python_node_results: tuple[PythonNodeExecutionResult, ...], - style: CliStyle, -) -> list[str]: - lines: list[str] = [] - has_warnings: bool = False - - model_result: ModelExecutionResult - for model_result in result.model_results: - model_warnings: list[str] = [] - audit_r: AuditExecutionResult - for audit_r in model_result.audit_results: - if audit_r.outcome == AuditOutcome.WARN: - name: str = audit_r.audit_name - if audit_r.attached_column_name: - name = f"{audit_r.audit_name} ({audit_r.attached_column_name})" - if audit_r.evaluation_mode == AuditEvaluationMode.MEASUREMENT: - model_warnings.append(f" audit {name} measured {audit_r.measured_value}") - else: - row_label: str = "row" if audit_r.row_count == 1 else "rows" - model_warnings.append( - f" audit {name} returned {audit_r.row_count} {row_label}" - ) - warning_msg: str - for warning_msg in model_result.warning_messages: - model_warnings.append(f" {warning_msg}") - if model_warnings: - if not has_warnings: - lines.append("") - lines.append(style.warning_strong("Warnings:")) - lines.append("") - has_warnings = True - lines.append(f" {model_result.model_name}") - line: str - for line in model_warnings: - lines.append(line) - lines.append("") - - labelled_warnings: tuple[tuple[str, tuple[str, ...]], ...] = ( - *( - (f"{function_result.function_name} ({function_result.function_kind})", messages) - for function_result in result.function_results - if (messages := function_result.warning_messages) - ), - *( - (f"{load_result.source_name} (loader {load_result.loader_name})", messages) - for load_result in result.load_results - if (messages := load_result.warning_messages) - ), - *( - (f"{python_result.node_name} ({python_result.kind.value})", messages) - for python_result in python_node_results - if (messages := python_result.warning_messages) - ), - ) - label: str - messages: tuple[str, ...] - for label, messages in labelled_warnings: - if not has_warnings: - lines.append("") - lines.append(style.warning_strong("Warnings:")) - lines.append("") - has_warnings = True - lines.append(f" {label}") - for warning_msg in messages: - lines.append(f" {warning_msg}") - lines.append("") - - return lines - - -def _inspection_relation_message(relation_name: str) -> str: - if relation_name.endswith("__delta"): - return f"delta table kept for inspection: {relation_name}" - return f"staging table kept for inspection: {relation_name}" - - -def _format_failure_error_block( - *, error_code: str | None, error_message: str, error_help: str | None, style: CliStyle -) -> list[str]: - lines: list[str] = [] - label_padding: str = " " * max(0, _TYPE_WIDTH - len("error")) - label: str = f"{style.error_muted('error')}{label_padding}" - message: str = _format_result_error( - error_code=error_code, - error_message=error_message, - error_help=error_help, - use_color=style.use_color, - ) - formatted_line: str - for index, formatted_line in enumerate(message.splitlines() or [message]): - display_label: str = label if index == 0 else " " * _TYPE_WIDTH - lines.append(f" {display_label}{formatted_line}") - return lines - - -def _format_result_error( - *, error_code: str | None, error_message: str, error_help: str | None, use_color: bool -) -> str: - if error_code is None: - return error_message - return format_coded_error( - code=error_code, - message=error_message, - help=error_help, - use_color=use_color, - include_error_label=False, - ) - - -def _format_duration(duration_ms: int | None) -> str: - if duration_ms is None: - return "" - seconds: float = duration_ms / 1000.0 - seconds_per_minute: int = 60 - if seconds < seconds_per_minute: - return f"{seconds:.2f}s" - minutes: int = int(seconds // 60) - remaining: float = seconds - minutes * 60 - return f"{minutes}m{remaining:.1f}s" - - -def _execution_status_display(status: ExecutionStatus) -> str: - if status == ExecutionStatus.SUCCESS: - return "OK" - if status == ExecutionStatus.FAILED: - return "FAIL" - if status == ExecutionStatus.SKIPPED: - return "SKIP" - return str(status) - - -def _load_result_detail(result: LoadExecutionResult) -> str: - if result.status != ExecutionStatus.SKIPPED: - return f" rows={result.rows_loaded:,}" - skip_label: str = "skip" - if result.skip_mode is not None: - skip_label = f"{result.skip_mode.value} skip" - if result.skip_reason: - return f" {skip_label}: {result.skip_reason}" - return f" {skip_label}" - - -def _model_skip_detail(result: ModelExecutionResult) -> str: - skip_label: str = "skip" - if result.skip_mode is not None: - skip_label = f"{result.skip_mode.value} skip" - if result.skip_reason: - return f" {skip_label}: {result.skip_reason}" - return f" {skip_label}" - - -def _hook_skip_detail(result: HookExecutionResult) -> str: - if result.status != ExecutionStatus.SKIPPED: - return "" - skip_label: str = "skip" - if result.skip_mode is not None: - skip_label = f"{result.skip_mode.value} skip" - if result.skip_reason: - return f" {skip_label}: {result.skip_reason}" - return f" {skip_label}" - - -def _audit_outcome_display(outcome: AuditOutcome) -> str: - if outcome == AuditOutcome.PASS: - return "PASS" - if outcome == AuditOutcome.WARN: - return "WARN" - if outcome == AuditOutcome.ERROR: - return "FAIL" - if outcome == AuditOutcome.INSUFFICIENT: - return "INSUFFICIENT" - return str(outcome) - - -def _test_outcome_display(outcome: SqlTestOutcome) -> str: - if outcome == SqlTestOutcome.PASS: - return "PASS" - if outcome == SqlTestOutcome.ERROR: - return "ERROR" - return "FAIL" - - -def _aggregate_audit_results( - audit_results: tuple[AuditExecutionResult, ...], -) -> list[AuditDisplayEntry]: - """Aggregate per-batch delta audit results into single display entries.""" - - has_delta: bool = any(a.run_scope_phase == AuditRunScope.DELTA_AND_FINAL for a in audit_results) - - groups: dict[tuple[str, str | None, str], list[AuditExecutionResult]] = {} - audit: AuditExecutionResult - for audit in audit_results: - key: tuple[str, str | None, str] = ( - audit.audit_name, - audit.attached_column_name, - audit.run_scope_phase, - ) - groups.setdefault(key, []).append(audit) - - entries: list[AuditDisplayEntry] = [] - results: list[AuditExecutionResult] - for (_name, _col, _phase), results in groups.items(): - display_name: str = _name - if _col is not None: - display_name = f"{_name} ({_col})" - worst: AuditOutcome = _worst_audit_outcome(results) - total_rows: int = sum(r.row_count for r in results) - pass_count: int = sum(1 for r in results if r.outcome == AuditOutcome.PASS) - reused: bool = all(r.reused for r in results) - label: str = _phase_label( - phase=_phase, has_delta_audits=has_delta, batch_count=len(results) - ) - entries.append( - AuditDisplayEntry( - label=label, - display_name=display_name, - outcome=worst, - total_row_count=total_rows, - batch_pass=pass_count, - batch_total=len(results), - reused=reused, - executed_sql=results[0].executed_sql if results else None, - ) - ) - - return entries - - -def _worst_audit_outcome(results: list[AuditExecutionResult]) -> AuditOutcome: - """Return the worst outcome from a list of audit results.""" - - has_error: bool = any(r.outcome == AuditOutcome.ERROR for r in results) - if has_error: - return AuditOutcome.ERROR - has_warn: bool = any(r.outcome == AuditOutcome.WARN for r in results) - if has_warn: - return AuditOutcome.WARN - if any(r.outcome == AuditOutcome.INSUFFICIENT for r in results): - return AuditOutcome.INSUFFICIENT - return AuditOutcome.PASS - - -def _phase_label(*, phase: str, has_delta_audits: bool, batch_count: int) -> str: - """Return the audit type label, annotated with phase when delta audits are present.""" - - if not has_delta_audits: - return "audit" - if phase == AuditRunScope.DELTA_AND_FINAL: - return "audit (d)" - return "audit (f)" - - -def _resolve_verbose_events( - *, model_result: ModelExecutionResult, plan_entry: ModelPlanEntry | None -) -> tuple[LifeCycleEvent, ...]: - if model_result.lifecycle_events: - return model_result.lifecycle_events - if plan_entry is not None: - return (LifeCycleEvent(kind=LifeCycleEventKind.SQL, content=plan_entry.logical_ddl),) - return () - - -def _hook_label_width(hook_results: tuple[HookExecutionResult, ...]) -> int: - if not hook_results: - return _HOOK_MIN_LABEL_WIDTH - longest_label: int = max(len(result.label) for result in hook_results) - return max(_HOOK_MIN_LABEL_WIDTH, min(longest_label, _HOOK_MAX_LABEL_WIDTH)) - - -def _scheduler_diagnostic_message(state: SchedulerState) -> str: - if state.aborted: - return ( - "Scheduler " - f"{state.running} running, {state.ready} ready, " - f"{state.waiting} waiting on dependencies, {state.aborted} aborted after stop, " - f"limit {state.limit}" - ) - suffix: str = ( - " (DAG frontier constrained)" - if state.running < state.limit and state.ready == 0 and state.waiting > 0 - else "" - ) - return ( - "Scheduler " - f"{state.running} running, {state.ready} ready, " - f"{state.waiting} waiting on dependencies, limit {state.limit}{suffix}" - ) - - -def _query_diagnostic_message(telemetry: StatementExecutionTelemetry) -> str: - query_id: str = telemetry.query_id or "query ID unavailable" - return ( - "Query " - f"{telemetry.resource_type} {telemetry.resource_name} phase={telemetry.phase} " - f"{query_id} {telemetry.status} {telemetry.elapsed_seconds:.2f}s" - ) - - -def _format_display_sql(sql: str) -> str: - stripped: str = sql.rstrip() - if not stripped: - return sql - if stripped.endswith(";"): - return stripped - return f"{stripped};" - - -def _tree_child_name_length(name: str) -> int: - """Measure a tree child name so its status still fits the shared status column.""" - - return len(name) + _TREE_CHILD_NAME_INDENT - _NAME_PADDING - - -def _truncate_name(*, name: str, width: int) -> str: - """Truncate a display name to fit within the given width.""" - - if len(name) <= width: - return name - return name[: width - 3] + "..." - - -def _format_streaming_batch_summary(*, model_result: ModelExecutionResult) -> str | None: - """Return the batch summary line for streaming output, if applicable.""" - - if model_result.batch_count is None: - return None - batch_label: str = "batch" if model_result.batch_count == 1 else "batches" - count_text: str = f"{model_result.batch_count} {batch_label}" - if model_result.batch_size is not None: - count_text = f"{count_text} ({model_result.batch_size})" - parts: list[str] = [count_text] - if ( - model_result.cursor_range_start is not None - and model_result.cursor_range_end is not None - and not model_result.empty_cursor_inputs - ): - start: str = cursor_bound_display( - value=model_result.cursor_range_start, - cursor_type=model_result.cursor_type, - cursor_grain=model_result.cursor_grain, - ) - end: str = inclusive_cursor_end( - end=model_result.cursor_range_end, - cursor_type=model_result.cursor_type, - cursor_grain=model_result.cursor_grain, - ) - parts.append(f"range {start} \u2192 {end}") - if model_result.rows_affected is not None: - parts.append(_format_abbreviated_rows(count=model_result.rows_affected)) - if model_result.empty_cursor_inputs and not model_result.waiting_on_empty_inputs: - parts.append(format_no_input_rows(model_result.empty_cursor_inputs)) - if model_result.microbatch_recovery_batch_count: - parts.append(f"{model_result.microbatch_recovery_batch_count} recovery") - if model_result.microbatch_synthetic_completion_count: - parts.append(f"{model_result.microbatch_synthetic_completion_count} synthetic coverage") - if model_result.microbatch_replay_requirement_state is not None: - parts.append(f"replay {model_result.microbatch_replay_requirement_state}") - return f" {' '.join(parts)}" - - -def _format_empty_cursor_inputs(*, model_result: ModelExecutionResult) -> str | None: - """Return why a cursor window selected no rows, if its inputs were empty.""" - - if not model_result.empty_cursor_inputs or model_result.waiting_on_empty_inputs: - return None - return f" {format_no_input_rows(model_result.empty_cursor_inputs)}" - - -def _format_abbreviated_rows(*, count: int) -> str: - """Format a row count with K/M/B abbreviation above 10,000.""" - - label: str = "row" if count == 1 else "rows" - if count >= _ROW_COUNT_BILLION: - return f"{count / _ROW_COUNT_BILLION:.1f}B {label}" - if count >= _ROW_COUNT_MILLION: - return f"{count / _ROW_COUNT_MILLION:.1f}M {label}" - if count >= _ROW_COUNT_ABBREVIATION_THRESHOLD: - return f"{count / _ROW_COUNT_THOUSAND:.1f}K {label}" - return f"{count:,} {label}" diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 30b98977e7..cab2bdd5c2 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -29,7 +29,6 @@ SkillUpdateResult, WrittenTarget, ) -from sqlbuild.cli.progress.models import AuditDisplayEntry, ExecutionCounts # noqa: F401 from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.cost.types import CostStatus from sqlbuild.spec.contracts.models import ( diff --git a/src/sqlbuild/cli/progress/models.py b/src/sqlbuild/cli/progress/models.py index cbefa6e379..91c51b4c99 100644 --- a/src/sqlbuild/cli/progress/models.py +++ b/src/sqlbuild/cli/progress/models.py @@ -5,7 +5,6 @@ from dataclasses import dataclass from sqlbuild._native import ConnectionProgressReporter, PlanningProgressReporter -from sqlbuild.compiler.auditing.types import AuditOutcome @dataclass(frozen=True) @@ -24,31 +23,3 @@ class NestedProgressChildRow: name: str status_text: str detail: str = "" - - -@dataclass(frozen=True) -class AuditDisplayEntry: - """Aggregated audit result for display.""" - - label: str - display_name: str - outcome: AuditOutcome - total_row_count: int - batch_pass: int - batch_total: int - reused: bool = False - executed_sql: str | None = None - - -@dataclass(frozen=True) -class ExecutionCounts: - """Aggregated pass, warning, failure, and skip counts.""" - - pass_count: int = 0 - warn_count: int = 0 - fail_count: int = 0 - skip_count: int = 0 - - @property - def total_count(self) -> int: - return self.pass_count + self.warn_count + self.fail_count + self.skip_count diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 49e8dfc4cc..220e1bf020 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -108,7 +108,7 @@ from sqlbuild.executor.seed.main._execute import execute_seed from sqlbuild.executor.testing.constants import SQL_TEST_ENTRY_MISSING_CODE from sqlbuild.executor.testing.main._execute import execute_sql_test -from sqlbuild.executor.testing.main.resource_id import sql_test_resource_id +from sqlbuild.executor.testing.main._resource_id import sql_test_resource_id from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome from sqlbuild.microbatches.classes.direct_store import ( diff --git a/src/sqlbuild/executor/pipeline/_helpers/testing.py b/src/sqlbuild/executor/pipeline/_helpers/testing.py index 152c03702c..bf5ebd668b 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/testing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/testing.py @@ -31,7 +31,7 @@ from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.constants import SQL_TEST_EXECUTION_ERROR_CODE from sqlbuild.executor.testing.main._execute import execute_sql_test -from sqlbuild.executor.testing.main.resource_id import sql_test_resource_id +from sqlbuild.executor.testing.main._resource_id import sql_test_resource_id from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.executor.testing.types import SqlTestOutcome from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/executor/testing/main/resource_id.py b/src/sqlbuild/executor/testing/main/_resource_id.py similarity index 100% rename from src/sqlbuild/executor/testing/main/resource_id.py rename to src/sqlbuild/executor/testing/main/_resource_id.py diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py index 36668c4da2..684c13669f 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -113,6 +113,7 @@ def test_given_native_source_when_checking_module_paths_then_reports_errors( "sqlbuild.cli.commands._helpers.cost.collection", "sqlbuild.cli.commands.classes.prepared_compile_artifacts", "sqlbuild.compile", + "sqlbuild.executor.runtime", "sqlbuild.fingerprints", "sqlbuild.lineage", ), diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py index 4be4a9b530..1c03c3f7a8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py @@ -1,34 +1,12 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.auditing.types import AuditOutcome from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.runtime.contracts.types import ExecutionResourceKind -@dataclass(frozen=True) -class TruncateNameTestCase: - description: str - name: str - width: int - expected_result: str - - -@dataclass(frozen=True) -class AuditAggregationTestCase: - description: str - audit_results: tuple[AuditExecutionResult, ...] - expected_entry_count: int - expected_labels: tuple[str, ...] - expected_outcomes: tuple[AuditOutcome, ...] - expected_batch_totals: tuple[int, ...] - expected_batch_passes: tuple[int, ...] - expected_reused: tuple[bool, ...] = field(default_factory=tuple) - - @dataclass(frozen=True) class BuildFooterTestCase: description: str diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py index fd42c3c6c9..39e0dbcb10 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py @@ -3,8 +3,6 @@ from __future__ import annotations import re -import threading -from collections.abc import Callable from pathlib import Path from sqlbuild.compiler.auditing.types import ( @@ -32,15 +30,6 @@ ) -def write_spinner_line_and_release( - *, - write_spinner_line: Callable[[], None], - spinner_updates: threading.Semaphore, -) -> None: - write_spinner_line() - spinner_updates.release() - - def build_audit_result( *, name: str, diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py index 9d73059397..0be484741e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py @@ -2,30 +2,24 @@ from __future__ import annotations -import threading -from collections.abc import Callable +import time from functools import partial from io import StringIO import pytest from sqlbuild._native import ( + BuildProgressCallbacks, NestedCommandProgressCallbacks, build_test_expectation_rows, + format_build_footer, resolve_test_name_width, write_execution_header, ) from sqlbuild.adapter.contract.models import LifeCycleEvent from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.commands.classes import build_progress_callbacks -from sqlbuild.cli.commands.classes.build_progress_callbacks import ( - BuildProgressCallbacks, - _aggregate_audit_results, - _truncate_name, - format_build_footer, -) -from sqlbuild.cli.progress.models import AuditDisplayEntry, NestedProgressChildRow -from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope +from sqlbuild.cli.progress.models import NestedProgressChildRow +from sqlbuild.compiler.auditing.types import AuditOutcome from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cost.models import StatementExecutionTelemetry @@ -42,9 +36,9 @@ from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.executor.testing.types import SqlTestOutcome +from sqlbuild.presentation.main import terminal_columns as terminal_columns_module from sqlbuild.runtime.contracts.types import ExecutionResourceKind from tests.unit.src.sqlbuild.cli.commands.shared._helpers._test_types import ( - AuditAggregationTestCase, BuildFooterTestCase, BuildProgressActiveSpinnerTestCase, BuildProgressFailureOutputTestCase, @@ -58,7 +52,6 @@ NestedProgressConcurrentTestCase, RuntimeDiagnosticsTestCase, SqlTestStatusAlignmentTestCase, - TruncateNameTestCase, ) from tests.unit.src.sqlbuild.cli.commands.shared._helpers.helpers import ( build_audit_result, @@ -66,7 +59,6 @@ build_sql_test_plan_entry, build_sql_test_result, status_columns, - write_spinner_line_and_release, ) @@ -199,44 +191,6 @@ def test_given_terminal_scheduler_state_when_reporting_then_output_explains_abor assert absent_fragment not in output -@pytest.mark.parametrize( - "test_case", - [ - TruncateNameTestCase( - description="name shorter than width is returned unchanged", - name="orders", - width=20, - expected_result="orders", - ), - TruncateNameTestCase( - description="name exactly at width is returned unchanged", - name="orders", - width=6, - expected_result="orders", - ), - TruncateNameTestCase( - description="name longer than width is truncated with ellipsis", - name="very_long_model_name_that_exceeds_limit", - width=20, - expected_result="very_long_model_n...", - ), - TruncateNameTestCase( - description="name one char over width truncates correctly", - name="abcdefgh", - width=7, - expected_result="abcd...", - ), - ], - ids=lambda case: case.description, -) -def test_given_name_and_width_when_truncating_then_returns_expected_result( - test_case: TruncateNameTestCase, -) -> None: - result: str = _truncate_name(name=test_case.name, width=test_case.width) - - assert result == test_case.expected_result - - @pytest.mark.parametrize( "test_case", [ @@ -350,194 +304,6 @@ def test_given_overlapping_items_when_completing_out_of_order_then_headers_follo assert stream.getvalue() == test_case.expected_output -@pytest.mark.parametrize( - "test_case", - [ - AuditAggregationTestCase( - description="final-only audits produce one entry per audit with audit label", - audit_results=( - build_audit_result(name="not_null", outcome=AuditOutcome.PASS, column_name="id"), - build_audit_result(name="unique", outcome=AuditOutcome.PASS, column_name="id"), - ), - expected_entry_count=2, - expected_labels=("audit", "audit"), - expected_outcomes=(AuditOutcome.PASS, AuditOutcome.PASS), - expected_batch_totals=(1, 1), - expected_batch_passes=(1, 1), - expected_reused=(False, False), - ), - AuditAggregationTestCase( - description="reused final audit records reused display flag", - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - reused=True, - ), - ), - expected_entry_count=1, - expected_labels=("audit",), - expected_outcomes=(AuditOutcome.PASS,), - expected_batch_totals=(1,), - expected_batch_passes=(1,), - expected_reused=(True,), - ), - AuditAggregationTestCase( - description="delta_and_final audits produce separate delta and final entries", - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.FINAL, - ), - ), - expected_entry_count=2, - expected_labels=("audit (d)", "audit (f)"), - expected_outcomes=(AuditOutcome.PASS, AuditOutcome.PASS), - expected_batch_totals=(1, 1), - expected_batch_passes=(1, 1), - expected_reused=(False, False), - ), - AuditAggregationTestCase( - description="multiple delta batches aggregate into one entry with batch count", - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.FINAL, - ), - ), - expected_entry_count=2, - expected_labels=("audit (d)", "audit (f)"), - expected_outcomes=(AuditOutcome.PASS, AuditOutcome.PASS), - expected_batch_totals=(3, 1), - expected_batch_passes=(3, 1), - expected_reused=(False, False), - ), - AuditAggregationTestCase( - description="worst outcome across batches is reported when one batch fails", - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.ERROR, - column_name="id", - row_count=5, - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.FINAL, - ), - ), - expected_entry_count=2, - expected_labels=("audit (d)", "audit (f)"), - expected_outcomes=(AuditOutcome.ERROR, AuditOutcome.PASS), - expected_batch_totals=(3, 1), - expected_batch_passes=(2, 1), - expected_reused=(False, False), - ), - AuditAggregationTestCase( - description="warn is worst outcome when no errors present", - audit_results=( - build_audit_result( - name="row_check", - outcome=AuditOutcome.PASS, - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="row_check", - outcome=AuditOutcome.WARN, - row_count=2, - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="row_check", - outcome=AuditOutcome.PASS, - run_scope_phase=AuditRunScope.FINAL, - ), - ), - expected_entry_count=2, - expected_labels=("audit (d)", "audit (f)"), - expected_outcomes=(AuditOutcome.WARN, AuditOutcome.PASS), - expected_batch_totals=(2, 1), - expected_batch_passes=(1, 1), - expected_reused=(False, False), - ), - ], - ids=lambda case: case.description, -) -def test_given_audit_results_when_aggregating_then_produces_expected_entries( - test_case: AuditAggregationTestCase, -) -> None: - entries: list[AuditDisplayEntry] = _aggregate_audit_results(test_case.audit_results) - - assert len(entries) == test_case.expected_entry_count - - idx: int - for idx in range(test_case.expected_entry_count): - assert entries[idx].label == test_case.expected_labels[idx], ( - f"entry {idx}: expected label {test_case.expected_labels[idx]!r}, " - f"got {entries[idx].label!r}" - ) - assert entries[idx].outcome == test_case.expected_outcomes[idx], ( - f"entry {idx}: expected outcome {test_case.expected_outcomes[idx]!r}, " - f"got {entries[idx].outcome!r}" - ) - assert entries[idx].batch_total == test_case.expected_batch_totals[idx], ( - f"entry {idx}: expected batch_total {test_case.expected_batch_totals[idx]}, " - f"got {entries[idx].batch_total}" - ) - assert entries[idx].batch_pass == test_case.expected_batch_passes[idx], ( - f"entry {idx}: expected batch_pass {test_case.expected_batch_passes[idx]}, " - f"got {entries[idx].batch_pass}" - ) - assert entries[idx].reused == test_case.expected_reused[idx], ( - f"entry {idx}: expected reused {test_case.expected_reused[idx]}, " - f"got {entries[idx].reused}" - ) - - @pytest.mark.parametrize( "test_case", [ @@ -930,7 +696,7 @@ def test_given_failed_top_level_node_when_reporting_progress_then_writes_error_d stream: StringIO = StringIO() monkeypatch.setattr("sys.stdout", stream) monkeypatch.setattr( - build_progress_callbacks, + terminal_columns_module, "terminal_columns", partial(int, test_case.terminal_width), ) @@ -1290,23 +1056,10 @@ def test_given_active_top_level_node_when_waiting_then_spinner_advances_frames( plan=PlanOutput(), use_color=False, ) - spinner_updates: threading.Semaphore = threading.Semaphore(value=0) - write_spinner_line: Callable[[], None] = callbacks._write_spinner_line - monkeypatch.setattr( - callbacks, - "_write_spinner_line", - partial( - write_spinner_line_and_release, - write_spinner_line=write_spinner_line, - spinner_updates=spinner_updates, - ), - ) callbacks.on_node_start(name=test_case.node_name, resource_kind=test_case.node_type) try: - _spinner_frame: str - for _spinner_frame in test_case.expected_spinner_frames: - assert spinner_updates.acquire(timeout=test_case.timeout_seconds) + time.sleep(test_case.timeout_seconds) finally: callbacks.on_node_complete( ModelExecutionResult( diff --git a/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py b/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py index 7cd228ed06..35846a539d 100644 --- a/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py +++ b/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py @@ -8,11 +8,11 @@ import pytest from sqlbuild._native import ( + BuildProgressCallbacks, NativeProgressProjector, TerminalEventIndex, terminal_event_index_scope, ) -from sqlbuild.cli.commands.classes.build_progress_callbacks import BuildProgressCallbacks from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry From 1b8e3b14916a7721f54a2d8efffb800747c18ad8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:57:07 +0100 Subject: [PATCH 04/40] test: add a stateful planner differential harness --- Makefile | 14 +- .../_helpers/compile_driver/pipeline.rs | 2 + .../_helpers/frontier/stage_capture.rs | 32 +++ scripts/planner_differential/README.md | 71 +++++ scripts/planner_differential/__init__.py | 1 + .../planner_differential/_helpers/__init__.py | 0 .../_helpers/baseline/__init__.py | 1 + .../_helpers/baseline/baseline.py | 91 +++++++ .../_helpers/capturing/__init__.py | 1 + .../_helpers/capturing/canonical.py | 95 +++++++ .../_helpers/capturing/state_tables.py | 117 +++++++++ .../_helpers/corpus/__init__.py | 1 + .../_helpers/corpus/corpus.py | 75 ++++++ .../_helpers/running/__init__.py | 1 + .../_helpers/running/commands.py | 56 ++++ .../_helpers/running/options.py | 79 ++++++ .../_helpers/running/project_run.py | 227 ++++++++++++++++ .../_helpers/running/report.py | 38 +++ .../_helpers/running/run.py | 74 ++++++ .../_helpers/running/snapshot.py | 37 +++ .../_helpers/steps/__init__.py | 1 + .../_helpers/steps/arguments.py | 57 ++++ .../_helpers/steps/model_file.py | 50 ++++ .../_helpers/steps/mutations.py | 245 ++++++++++++++++++ .../_helpers/steps/relations.py | 12 + .../_helpers/steps/targets.py | 107 ++++++++ scripts/planner_differential/constants.py | 138 ++++++++++ scripts/planner_differential/main/__init__.py | 1 + .../planner_differential/main/differential.py | 37 +++ scripts/planner_differential/models.py | 116 +++++++++ scripts/run_planner_differential.py | 15 ++ .../scripts/planner_differential/__init__.py | 0 .../planner_differential/_test_types.py | 20 ++ .../scripts/planner_differential/helpers.py | 44 ++++ .../test_planner_differential.py | 64 +++++ .../scripts/planner_differential/__init__.py | 0 .../planner_differential/_helpers/__init__.py | 0 .../_helpers/capturing/__init__.py | 0 .../_helpers/capturing/_test_types.py | 42 +++ .../_helpers/capturing/helpers.py | 59 +++++ .../_helpers/capturing/test_canonical.py | 206 +++++++++++++++ .../_helpers/steps/__init__.py | 0 .../_helpers/steps/_test_types.py | 35 +++ .../_helpers/steps/test_model_file.py | 108 ++++++++ 44 files changed, 2369 insertions(+), 1 deletion(-) create mode 100644 scripts/planner_differential/README.md create mode 100644 scripts/planner_differential/__init__.py create mode 100644 scripts/planner_differential/_helpers/__init__.py create mode 100644 scripts/planner_differential/_helpers/baseline/__init__.py create mode 100644 scripts/planner_differential/_helpers/baseline/baseline.py create mode 100644 scripts/planner_differential/_helpers/capturing/__init__.py create mode 100644 scripts/planner_differential/_helpers/capturing/canonical.py create mode 100644 scripts/planner_differential/_helpers/capturing/state_tables.py create mode 100644 scripts/planner_differential/_helpers/corpus/__init__.py create mode 100644 scripts/planner_differential/_helpers/corpus/corpus.py create mode 100644 scripts/planner_differential/_helpers/running/__init__.py create mode 100644 scripts/planner_differential/_helpers/running/commands.py create mode 100644 scripts/planner_differential/_helpers/running/options.py create mode 100644 scripts/planner_differential/_helpers/running/project_run.py create mode 100644 scripts/planner_differential/_helpers/running/report.py create mode 100644 scripts/planner_differential/_helpers/running/run.py create mode 100644 scripts/planner_differential/_helpers/running/snapshot.py create mode 100644 scripts/planner_differential/_helpers/steps/__init__.py create mode 100644 scripts/planner_differential/_helpers/steps/arguments.py create mode 100644 scripts/planner_differential/_helpers/steps/model_file.py create mode 100644 scripts/planner_differential/_helpers/steps/mutations.py create mode 100644 scripts/planner_differential/_helpers/steps/relations.py create mode 100644 scripts/planner_differential/_helpers/steps/targets.py create mode 100644 scripts/planner_differential/constants.py create mode 100644 scripts/planner_differential/main/__init__.py create mode 100644 scripts/planner_differential/main/differential.py create mode 100644 scripts/planner_differential/models.py create mode 100644 scripts/run_planner_differential.py create mode 100644 tests/e2e/scripts/planner_differential/__init__.py create mode 100644 tests/e2e/scripts/planner_differential/_test_types.py create mode 100644 tests/e2e/scripts/planner_differential/helpers.py create mode 100644 tests/e2e/scripts/planner_differential/test_planner_differential.py create mode 100644 tests/unit/scripts/planner_differential/__init__.py create mode 100644 tests/unit/scripts/planner_differential/_helpers/__init__.py create mode 100644 tests/unit/scripts/planner_differential/_helpers/capturing/__init__.py create mode 100644 tests/unit/scripts/planner_differential/_helpers/capturing/_test_types.py create mode 100644 tests/unit/scripts/planner_differential/_helpers/capturing/helpers.py create mode 100644 tests/unit/scripts/planner_differential/_helpers/capturing/test_canonical.py create mode 100644 tests/unit/scripts/planner_differential/_helpers/steps/__init__.py create mode 100644 tests/unit/scripts/planner_differential/_helpers/steps/_test_types.py create mode 100644 tests/unit/scripts/planner_differential/_helpers/steps/test_model_file.py diff --git a/Makefile b/Makefile index ce32a1d665..1882629b73 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SHELL := /bin/bash test-e2e-cold-compile-performance test-e2e-cache-compile-performance \ test-e2e-dense-compile-performance test-e2e-varied-cache-performance \ compiler-differential compiler-differential-dense \ - compiler-baselines + compiler-baselines planner-differential format: uv run ruff format . @@ -145,6 +145,7 @@ E2E_DUCKDB_CLI_PATHS := \ tests/e2e/scripts/cli_preview \ tests/e2e/scripts/compile_performance_ratio \ tests/e2e/scripts/compiler_differential \ + tests/e2e/scripts/planner_differential \ tests/e2e/scripts/release_performance \ tests/e2e/src/sqlbuild/cli/commands/main/adapters \ tests/e2e/src/sqlbuild/cli/commands/main/audit \ @@ -326,6 +327,17 @@ compiler-differential-dense: --corpus dense --dense-models $(COMPILER_DIFFERENTIAL_DENSE_MODELS) \ --jobs $(COMPILER_DIFFERENTIAL_JOBS) $(COMPILER_DIFFERENTIAL_ARGS) +# Stateful planner differential (see scripts/planner_differential/README.md): record a +# baseline on the tree before a planner change, then check the changed tree against it. +PLANNER_DIFFERENTIAL_MODE ?= check +PLANNER_DIFFERENTIAL_JOBS ?= 2 +PLANNER_DIFFERENTIAL_SEEDS ?= 6 +PLANNER_DIFFERENTIAL_ARGS ?= +planner-differential: + env PYTHONUNBUFFERED=1 uv run python -m scripts.run_planner_differential \ + $(PLANNER_DIFFERENTIAL_MODE) --jobs $(PLANNER_DIFFERENTIAL_JOBS) \ + --seeds $(PLANNER_DIFFERENTIAL_SEEDS) $(PLANNER_DIFFERENTIAL_ARGS) + skills: uv run python -m scripts.generate_docs_skill diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index ffb6416eb8..1c8c6a3be5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -11,6 +11,7 @@ use crate::bindings::_helpers::compile_driver::pipeline_entries::{ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; +use crate::bindings::_helpers::frontier::stage_capture::write_plan_capture; const NATIVE_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; @@ -390,6 +391,7 @@ fn planned<'py>( tracker.setattr("planning_seconds", planning_seconds)?; } let result: Bound<'py, PyAny> = result?; + write_plan_capture(&result.getattr("plan_output")?)?; imported(py, "dataclasses", "replace")?.call( (result,), Some(&keywords( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs index dec05b79b1..8b838282a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs @@ -14,6 +14,8 @@ use crate::bindings::_helpers::frontier::encoder::{CaptureEncoder, emit_error}; const CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR"; const CAPTURE_SUFFIX: &str = ".json"; +const PLAN_CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_PLANNER_CAPTURE_DIR"; +const PLAN_CAPTURE_NAME: &str = "plan_output"; const FRONTIER_CONSTANTS_MODULE: &str = "sqlbuild.compiler.frontier.constants"; /// The capture document of `value`, or the shared nodes written before encoding failed. @@ -55,6 +57,36 @@ pub(crate) fn write_stage_capture( "__truediv__", (format!("{number:03}-{stage_name}{CAPTURE_SUFFIX}"),), )?; + write_capture_file(&path, value) +} + +/// Write the planner's `PlanOutput` when SQLBUILD_PLANNER_CAPTURE_DIR names a directory. +pub(crate) fn write_plan_capture(plan_output: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = plan_output.py(); + let directory: Bound<'_, PyAny> = + imported(py, "os", "environ")?.call_method1("get", (PLAN_CAPTURE_DIR_ENV_VAR,))?; + if !directory.is_truthy()? { + return Ok(()); + } + let number: u64 = imported(py, "builtins", "next")? + .call1((imported( + py, + FRONTIER_CONSTANTS_MODULE, + "STAGE_CAPTURE_SEQUENCE", + )?,))? + .extract()?; + let path: Bound<'_, PyAny> = imported(py, "pathlib", "Path")? + .call1((directory,))? + .call_method1( + "__truediv__", + (format!("{number:03}-{PLAN_CAPTURE_NAME}{CAPTURE_SUFFIX}"),), + )?; + write_capture_file(&path, plan_output) +} + +/// Write the capture document of `value` to `path`, creating its directory. +fn write_capture_file(path: &Bound<'_, PyAny>, value: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = path.py(); call_with( &path.getattr("parent")?.getattr("mkdir")?, &[ diff --git a/scripts/planner_differential/README.md b/scripts/planner_differential/README.md new file mode 100644 index 0000000000..582df4769f --- /dev/null +++ b/scripts/planner_differential/README.md @@ -0,0 +1,71 @@ +# Planner differential + +The compiler differential plans every project against an empty warehouse. The planner +differential checks the plans that depend on warehouse state: it builds each project, changes +it step by step, and records what the planner decides after every step. Use it around a change +that can alter planning (for example a port of planner code): record a baseline on the tree +before the change, then check the changed tree against it. + +```bash +# On the tree before the change (about 9 minutes with the defaults) +make planner-differential PLANNER_DIFFERENTIAL_MODE=record +# On the changed tree +make planner-differential +``` + +or call the script directly: + +```bash +uv run python -m scripts.run_planner_differential record --jobs 2 +uv run python -m scripts.run_planner_differential check --jobs 2 +uv run python -m scripts.run_planner_differential check --projects fixture/waffle_shop seed/0 +``` + +`check` exits 1 when any step output differs from the baseline or a project fails to run, and +prints a unified diff for every differing artifact. It writes the run it checked beside the +baseline (`.actual/`) for closer comparison. + +## Corpus + +The DuckDB fixture and example projects, then the first `--seeds` (default 6) generated seed +projects that are expected to compile and use the built-in DuckDB adapter. Generated seeds are +not required to build cleanly: model failures are part of the recorded outputs. `--projects` +keeps projects whose name contains any given text and `--limit` keeps the first N. + +## Mutation script + +`STEPS` in `constants.py` is the script every project runs, in order. Each `PlannerStep` names an +optional mutation, the kind of model it needs (`role`, with `index` choosing among the +candidates in name order), and extra flags for `sqb plan` and `sqb build`. Flags may contain +`{model}`, `{start_cursor}` and `{end_cursor}`, filled from the target model and its built +relation. A step whose role has no model, or whose placeholders have no value, is recorded as +skipped. Targets come from the previous unselected `sqb plan --json`, so later steps follow +earlier renames. + +The script covers a no-op rebuild, a query edit, a config-only change, a new declared column, a +column rename and a model rename through `sqb rename` (`migrate_from`), an added model, a +relation dropped outside SQLBuild, source rows added with a later cursor value, and the +`--max-microbatches`, cursor override, `--exclude`, `--select` and `--full-refresh` flags. To +add a step, append a `PlannerStep`; a new kind of change adds a mutation function in +`_helpers/steps/mutations.py`. When a mutation makes planning fail, the step is recorded and the +project files are restored so later steps still run. `--steps` runs a subset (include `initial` +so later steps have targets). + +## What each step records + +- `sqb plan --json`: exit code, the document, and error lines; +- `sqb plan`: exit code and text; +- the planner's full `PlanOutput`, written as a canonical JSON capture when + `SQLBUILD_PLANNER_CAPTURE_DIR` names a directory (the same encoder as the compiler stage + captures); +- `sqb build`: exit code, summary line and error lines; +- the rows each `_sqlbuild*` state table gained and lost during the step. + +Run ids, invocation ids, event ids, timings, timestamps from the run date on (such as retention +expiry), and the run directories are masked. Mappings keyed by non-string values are sorted by +key. Baselines hold only masked values, so a baseline recorded earlier on the same day or on +another day compares cleanly unless planning changed. Plans that depend on the current time +beyond these masks would show up as differences; mask them in `_helpers/capturing/canonical.py`. + +Baselines are about 45 MB for the default corpus and live under +`/tmp/sqlbuild-planner-differential/` by default; they are not committed. diff --git a/scripts/planner_differential/__init__.py b/scripts/planner_differential/__init__.py new file mode 100644 index 0000000000..5263573d34 --- /dev/null +++ b/scripts/planner_differential/__init__.py @@ -0,0 +1 @@ +"""Stateful planner differential: plans and state across a fixed mutation script.""" diff --git a/scripts/planner_differential/_helpers/__init__.py b/scripts/planner_differential/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/planner_differential/_helpers/baseline/__init__.py b/scripts/planner_differential/_helpers/baseline/__init__.py new file mode 100644 index 0000000000..8011b59e34 --- /dev/null +++ b/scripts/planner_differential/_helpers/baseline/__init__.py @@ -0,0 +1 @@ +"""Record and check planner baselines.""" diff --git a/scripts/planner_differential/_helpers/baseline/baseline.py b/scripts/planner_differential/_helpers/baseline/baseline.py new file mode 100644 index 0000000000..49556e77e7 --- /dev/null +++ b/scripts/planner_differential/_helpers/baseline/baseline.py @@ -0,0 +1,91 @@ +"""Write a run as the baseline, or compare it with the recorded baseline.""" + +from __future__ import annotations + +import difflib +import json +from pathlib import Path + +from scripts.compiler_differential._helpers.comparing.compare import as_json_object +from scripts.planner_differential.constants import BASELINE_SUFFIX, MAX_DIFF_LINES +from scripts.planner_differential.models import ProjectDifference + +_STEP_KEY: str = "step" +_STEPS_KEY: str = "steps" +_MISSING: str = "" + + +def write_document(*, directory: Path, slug: str, document: dict[str, object]) -> Path: + """Write one project's run as stable, indented JSON.""" + + directory.mkdir(parents=True, exist_ok=True) + path: Path = directory / f"{slug}{BASELINE_SUFFIX}" + _ = path.write_text(_rendered(document) + "\n", encoding="utf-8", newline="\n") + return path + + +def read_document(*, directory: Path, slug: str) -> dict[str, object] | None: + """Return a recorded project run, or None when the baseline has none.""" + + path: Path = directory / f"{slug}{BASELINE_SUFFIX}" + if not path.is_file(): + return None + return as_json_object(json.loads(path.read_text(encoding="utf-8"))) + + +def compare_documents( + *, expected: dict[str, object], actual: dict[str, object] +) -> tuple[ProjectDifference, ...]: + """Return a readable diff for every step artifact that differs from the baseline.""" + + expected_steps: list[dict[str, object]] = _steps(expected) + actual_steps: list[dict[str, object]] = _steps(actual) + differences: list[ProjectDifference] = [] + for index in range(max(len(expected_steps), len(actual_steps))): + left: dict[str, object] = expected_steps[index] if index < len(expected_steps) else {} + right: dict[str, object] = actual_steps[index] if index < len(actual_steps) else {} + step: str = str(right.get(_STEP_KEY, left.get(_STEP_KEY, index))) + for artifact in (*left, *(key for key in right if key not in left)): + if left.get(artifact, _MISSING) == right.get(artifact, _MISSING): + continue + differences.append( + ProjectDifference( + step=step, + artifact=artifact, + diff=unified_diff( + expected=left.get(artifact, _MISSING), + actual=right.get(artifact, _MISSING), + ), + ) + ) + return tuple(differences) + + +def unified_diff(*, expected: object, actual: object) -> str: + """Return a bounded unified diff of two JSON values rendered one item per line.""" + + lines: list[str] = list( + difflib.unified_diff( + _rendered(expected).splitlines(), + _rendered(actual).splitlines(), + fromfile="baseline", + tofile="actual", + lineterm="", + n=2, + ) + ) + if len(lines) > MAX_DIFF_LINES: + omitted: int = len(lines) - MAX_DIFF_LINES + lines = [*lines[:MAX_DIFF_LINES], f"... {omitted} more diff lines"] + return "\n".join(lines) + + +def _steps(document: dict[str, object]) -> list[dict[str, object]]: + steps: object = document.get(_STEPS_KEY) + if not isinstance(steps, list): + return [] + return [mapping for step in steps if (mapping := as_json_object(step)) is not None] + + +def _rendered(value: object) -> str: + return json.dumps(value, indent=1, ensure_ascii=False) diff --git a/scripts/planner_differential/_helpers/capturing/__init__.py b/scripts/planner_differential/_helpers/capturing/__init__.py new file mode 100644 index 0000000000..ebf1112e2a --- /dev/null +++ b/scripts/planner_differential/_helpers/capturing/__init__.py @@ -0,0 +1 @@ +"""Capture and canonicalise plans and state rows.""" diff --git a/scripts/planner_differential/_helpers/capturing/canonical.py b/scripts/planner_differential/_helpers/capturing/canonical.py new file mode 100644 index 0000000000..574825e90f --- /dev/null +++ b/scripts/planner_differential/_helpers/capturing/canonical.py @@ -0,0 +1,95 @@ +"""Mask run-specific values so two runs of the same tree compare equal.""" + +from __future__ import annotations + +import json +import re +from datetime import date, timedelta + +from scripts.compiler_differential.classes.capture_file import expand_capture_text +from scripts.planner_differential.constants import ( + COMPACT_TIMESTAMP_PATTERN, + ELAPSED_TIME_MASK, + ELAPSED_TIME_PATTERN, + ISO_TIMESTAMP_PATTERN, + MAPPING_MARKER, + RECENT_TIMESTAMP_DAYS, + RECENT_TIMESTAMP_MASK, + VOLATILE_KEY_SUFFIXES, + VOLATILE_KEYS, + VOLATILE_VALUE_MASK, +) +from scripts.planner_differential.models import TextMasks +from sqlbuild._native import mask_capture_noise + + +def mask_text(*, text: str, masks: TextMasks) -> str: + """Mask invocation ids, run paths and timestamps near the run date.""" + + masked: str = mask_capture_noise(text) + for prefix, replacement in masks.replacements: + masked = masked.replace(prefix, replacement) + masked = ISO_TIMESTAMP_PATTERN.sub( + lambda match: _recent_or_kept(match=match, today=masks.today, iso=True), masked + ) + return COMPACT_TIMESTAMP_PATTERN.sub( + lambda match: _recent_or_kept(match=match, today=masks.today, iso=False), masked + ) + + +def mask_console_text(*, text: str, masks: TextMasks) -> str: + """Mask console output: the artifact noise plus elapsed times.""" + + return ELAPSED_TIME_PATTERN.sub(ELAPSED_TIME_MASK, mask_text(text=text, masks=masks)) + + +def canonical_value(*, value: object, masks: TextMasks) -> object: + """Return decoded JSON with volatile keys and run-specific text masked.""" + + if isinstance(value, dict): + canonical: dict[str, object] = { + str(key): ( + VOLATILE_VALUE_MASK + if item is not None and _volatile_key(str(key)) + else canonical_value(value=item, masks=masks) + ) + for key, item in value.items() + } + pairs: object = canonical.get(MAPPING_MARKER) + if len(canonical) == 1 and isinstance(pairs, list): + return {MAPPING_MARKER: sorted(pairs, key=_sort_key)} + return canonical + if isinstance(value, list): + return [canonical_value(value=item, masks=masks) for item in value] + if isinstance(value, str): + return mask_text(text=value, masks=masks) + return value + + +def canonical_plan_capture(*, text: str, masks: TextMasks) -> object: + """Expand a plan capture document's shared nodes and mask its volatile values.""" + + return canonical_value(value=expand_capture_text(text), masks=masks) + + +def _volatile_key(key: str) -> bool: + return key in VOLATILE_KEYS or key.endswith(VOLATILE_KEY_SUFFIXES) + + +def _sort_key(pair: object) -> str: + key: object = pair[0] if isinstance(pair, list) and pair else pair + return json.dumps(key, sort_keys=True, ensure_ascii=False) + + +def _recent_or_kept(*, match: re.Match[str], today: date, iso: bool) -> str: + day_text: str = match.group(1) + try: + day: date = ( + date.fromisoformat(day_text) + if iso + else date(int(day_text[:4]), int(day_text[4:6]), int(day_text[6:])) + ) + except ValueError: + return match.group(0) + recent: bool = day >= today - timedelta(days=RECENT_TIMESTAMP_DAYS) + return RECENT_TIMESTAMP_MASK if recent else match.group(0) diff --git a/scripts/planner_differential/_helpers/capturing/state_tables.py b/scripts/planner_differential/_helpers/capturing/state_tables.py new file mode 100644 index 0000000000..d29fc7c4d9 --- /dev/null +++ b/scripts/planner_differential/_helpers/capturing/state_tables.py @@ -0,0 +1,117 @@ +"""Read SQLBuild state-table rows from a project's DuckDB files.""" + +from __future__ import annotations + +import base64 +import binascii +import json +from collections import Counter +from datetime import date, datetime, time +from decimal import Decimal +from pathlib import Path + +import duckdb + +from scripts.planner_differential._helpers.capturing.canonical import canonical_value +from scripts.planner_differential.constants import ( + BASE64_COLUMN_SUFFIX, + DUCKDB_SUFFIX, + STATE_TABLE_PREFIX, +) +from scripts.planner_differential.models import TextMasks + +_STATE_TABLES_SQL: str = ( + "SELECT table_schema, table_name FROM information_schema.tables " + "WHERE table_type = 'BASE TABLE' AND starts_with(table_name, ?) " + "ORDER BY table_schema, table_name" +) + + +def database_files(project_dir: Path) -> tuple[Path, ...]: + """Return the project's DuckDB files in a stable order.""" + + return tuple(sorted(project_dir.rglob(f"*{DUCKDB_SUFFIX}"))) + + +def state_rows(*, project_dir: Path, masks: TextMasks) -> dict[str, list[str]]: + """Return each state table's masked rows as compact JSON, sorted.""" + + tables: dict[str, list[str]] = {} + for path in database_files(project_dir): + connection: duckdb.DuckDBPyConnection = duckdb.connect(str(path), read_only=True) + try: + names: list[tuple[str, str]] = connection.execute( + _STATE_TABLES_SQL, [STATE_TABLE_PREFIX] + ).fetchall() + for schema, table in names: + key: str = f"{path.relative_to(project_dir).as_posix()}:{schema}.{table}" + tables[key] = _table_rows( + connection=connection, relation=f'"{schema}"."{table}"', masks=masks + ) + finally: + connection.close() + return tables + + +def state_delta( + *, before: dict[str, list[str]], after: dict[str, list[str]] +) -> dict[str, dict[str, list[object]]]: + """Return the rows each state table gained and lost, as decoded JSON.""" + + delta: dict[str, dict[str, list[object]]] = {} + for table in sorted({*before, *after}): + old: Counter[str] = Counter(before.get(table, [])) + new: Counter[str] = Counter(after.get(table, [])) + added: list[str] = sorted((new - old).elements()) + removed: list[str] = sorted((old - new).elements()) + if added or removed: + delta[table] = { + "added": [json.loads(row) for row in added], + "removed": [json.loads(row) for row in removed], + } + return delta + + +def _table_rows( + *, connection: duckdb.DuckDBPyConnection, relation: str, masks: TextMasks +) -> list[str]: + cursor: duckdb.DuckDBPyConnection = connection.execute(f"SELECT * FROM {relation}") + columns: list[str] = [str(column[0]) for column in cursor.description or ()] + rows: list[str] = [] + for row in cursor.fetchall(): + decoded: dict[str, object] = { + column: _decoded(column=column, value=value) + for column, value in zip(columns, row, strict=True) + } + rows.append( + json.dumps( + canonical_value(value=decoded, masks=masks), + sort_keys=True, + ensure_ascii=False, + default=str, + ) + ) + return sorted(rows) + + +def _decoded(*, column: str, value: object) -> object: + if isinstance(value, datetime | date | time): + return value.isoformat() + if isinstance(value, Decimal): + return str(value) + if isinstance(value, bytes): + return value.hex() + if isinstance(value, str) and column.endswith(BASE64_COLUMN_SUFFIX): + return _base64_json(value) + return value + + +def _base64_json(value: str) -> object: + try: + text: str = base64.b64decode(value, validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError): + return value + try: + return json.loads(text) + except json.JSONDecodeError: + return text diff --git a/scripts/planner_differential/_helpers/corpus/__init__.py b/scripts/planner_differential/_helpers/corpus/__init__.py new file mode 100644 index 0000000000..3c565581d0 --- /dev/null +++ b/scripts/planner_differential/_helpers/corpus/__init__.py @@ -0,0 +1 @@ +"""Select the projects the planner differential runs.""" diff --git a/scripts/planner_differential/_helpers/corpus/corpus.py b/scripts/planner_differential/_helpers/corpus/corpus.py new file mode 100644 index 0000000000..2d432ae711 --- /dev/null +++ b/scripts/planner_differential/_helpers/corpus/corpus.py @@ -0,0 +1,75 @@ +"""Select the fixture, example and generated projects the planner differential runs.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +from scripts.compiler_differential.classes.project_builder import ProjectBuilder +from scripts.compiler_differential.models import GeneratedProject +from scripts.planner_differential.constants import ( + DUCKDB_ADAPTER, + EXAMPLE_ROOT, + FIXTURE_ROOT, + PROJECT_CONFIG_FILE, +) +from scripts.planner_differential.models import PlannerProject + +_SEED_SEARCH_FACTOR: int = 10 +_FIXTURE_KIND: str = "fixture" +_EXAMPLE_KIND: str = "example" + + +def build_corpus(*, repo_root: Path, seed_start: int, seeds: int) -> list[PlannerProject]: + """Return DuckDB fixtures and examples, then the first `seeds` valid built-in DuckDB seeds.""" + + projects: list[PlannerProject] = [] + for kind, root in ((_FIXTURE_KIND, FIXTURE_ROOT), (_EXAMPLE_KIND, EXAMPLE_ROOT)): + projects.extend( + PlannerProject(name=f"{kind}/{path.name}", source_dir=path) + for path in _duckdb_projects(repo_root / root) + ) + found: int = 0 + for seed in range(seed_start, seed_start + seeds * _SEED_SEARCH_FACTOR): + if found == seeds: + break + generated: GeneratedProject = ProjectBuilder(seed).build() + if generated.expected_error_code is not None or not _duckdb_config( + generated.files.get(PROJECT_CONFIG_FILE, "") + ): + continue + projects.append(PlannerProject(name=f"seed/{seed}", writer=generated.write)) + found += 1 + return projects + + +def filter_corpus( + *, projects: list[PlannerProject], names: tuple[str, ...], limit: int | None +) -> list[PlannerProject]: + """Keep projects whose name contains any filter, then the first `limit` of them.""" + + kept: list[PlannerProject] = [ + project for project in projects if not names or _matches(name=project.name, filters=names) + ] + return kept if limit is None else kept[:limit] + + +def _matches(*, name: str, filters: tuple[str, ...]) -> bool: + return any(item in name for item in filters) + + +def _duckdb_projects(root: Path) -> list[Path]: + return sorted( + path + for path in root.iterdir() + if path.is_dir() + and (path / PROJECT_CONFIG_FILE).is_file() + and _duckdb_config((path / PROJECT_CONFIG_FILE).read_text(encoding="utf-8")) + ) + + +def _duckdb_config(text: str) -> bool: + try: + return tomllib.loads(text).get("adapter") == DUCKDB_ADAPTER + except tomllib.TOMLDecodeError: + return False diff --git a/scripts/planner_differential/_helpers/running/__init__.py b/scripts/planner_differential/_helpers/running/__init__.py new file mode 100644 index 0000000000..5d7f937507 --- /dev/null +++ b/scripts/planner_differential/_helpers/running/__init__.py @@ -0,0 +1 @@ +"""Run one project through the mutation script.""" diff --git a/scripts/planner_differential/_helpers/running/commands.py b/scripts/planner_differential/_helpers/running/commands.py new file mode 100644 index 0000000000..44e315da88 --- /dev/null +++ b/scripts/planner_differential/_helpers/running/commands.py @@ -0,0 +1,56 @@ +"""Run sqb in a fresh process with the harness environment.""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +from scripts.compiler_differential.constants import GENERATOR_ENVIRONMENT +from scripts.planner_differential.constants import ( + COMMAND_TIMEOUT_SECONDS, + EXCLUDED_ENVIRONMENT_KEYS, + EXCLUDED_ENVIRONMENT_PREFIX, + SQB_ENTRY, +) +from scripts.planner_differential.models import CommandResult + +_TIMEOUT_EXIT_CODE: int = -1 + + +def harness_environment() -> dict[str, str]: + """Return this environment without capture or dbt settings, plus generator variables.""" + + return { + **{ + key: value + for key, value in os.environ.items() + if key not in EXCLUDED_ENVIRONMENT_KEYS + and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX) + }, + **GENERATOR_ENVIRONMENT, + } + + +def run_sqb( + *, arguments: tuple[str, ...], project_dir: Path, environment: dict[str, str], python: Path +) -> CommandResult: + """Run one sqb command in `project_dir`; a timeout reports exit code -1.""" + + try: + completed: subprocess.CompletedProcess[str] = subprocess.run( + [str(python), "-c", SQB_ENTRY, "--no-color", *arguments], + cwd=project_dir, + env=environment, + capture_output=True, + text=True, + encoding="utf-8", + errors="surrogateescape", + timeout=COMMAND_TIMEOUT_SECONDS, + check=False, + ) + except subprocess.TimeoutExpired: + return CommandResult(exit_code=_TIMEOUT_EXIT_CODE, stdout="", stderr="timed out") + return CommandResult( + exit_code=completed.returncode, stdout=completed.stdout, stderr=completed.stderr + ) diff --git a/scripts/planner_differential/_helpers/running/options.py b/scripts/planner_differential/_helpers/running/options.py new file mode 100644 index 0000000000..d09e560cd2 --- /dev/null +++ b/scripts/planner_differential/_helpers/running/options.py @@ -0,0 +1,79 @@ +"""Parse the planner differential command line into run options.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from scripts.planner_differential.constants import ( + ACTUAL_SUFFIX, + DEFAULT_BASELINE_DIR, + DEFAULT_JOBS, + DEFAULT_SEED_COUNT, + MODE_CHECK, + MODES, + STEPS, +) +from scripts.planner_differential.models import RunOptions + + +def parse_args(argv: list[str] | None) -> argparse.Namespace: + """Parse and validate the harness arguments.""" + + parser: argparse.ArgumentParser = argparse.ArgumentParser( + prog="python -m scripts.run_planner_differential", + description=( + "Build each corpus project, apply a fixed mutation script, and record or check " + "plans, plan outputs and state-table rows after every step." + ), + ) + parser.add_argument("mode", choices=MODES, help="record a baseline, or check against it") + parser.add_argument( + "--baseline-dir", + type=Path, + default=Path(DEFAULT_BASELINE_DIR), + help=( + f"baseline directory (default: {DEFAULT_BASELINE_DIR}); check writes the run " + f"beside it with an {ACTUAL_SUFFIX} suffix" + ), + ) + parser.add_argument("--jobs", type=int, default=DEFAULT_JOBS) + parser.add_argument("--seeds", type=int, default=DEFAULT_SEED_COUNT) + parser.add_argument("--seed-start", type=int, default=0) + parser.add_argument( + "--projects", + nargs="*", + default=[], + help="keep projects whose name contains any of these, such as fixture/ or seed/3", + ) + parser.add_argument( + "--steps", + nargs="*", + choices=[step.name for step in STEPS], + default=[], + help="run only these steps of the mutation script, in script order", + ) + parser.add_argument("--limit", type=int, default=None, help="run only the first N projects") + parser.add_argument( + "--work-dir", type=Path, default=None, help="keep project copies and captures here" + ) + parser.add_argument("--python", type=Path, default=Path(sys.executable)) + args: argparse.Namespace = parser.parse_args(argv) + if args.mode == MODE_CHECK and not args.baseline_dir.is_dir(): + parser.error(f"no baseline at {args.baseline_dir}; run with `record` first") + return args + + +def run_options(*, args: argparse.Namespace, work_dir: Path) -> RunOptions: + """Return run options from parsed arguments, running in `work_dir`.""" + + return RunOptions( + work_dir=work_dir.absolute(), + jobs=args.jobs, + python=args.python, + mode=args.mode, + baseline_dir=args.baseline_dir.absolute(), + keep_work=args.work_dir is not None, + steps=tuple(step for step in STEPS if not args.steps or step.name in args.steps), + ) diff --git a/scripts/planner_differential/_helpers/running/project_run.py b/scripts/planner_differential/_helpers/running/project_run.py new file mode 100644 index 0000000000..0a39cd4c67 --- /dev/null +++ b/scripts/planner_differential/_helpers/running/project_run.py @@ -0,0 +1,227 @@ +"""Run one project through the mutation script and record every step's outputs.""" + +from __future__ import annotations + +import json +import shutil +from datetime import date +from pathlib import Path + +from scripts.compiler_differential._helpers.comparing.compare import as_json_object +from scripts.planner_differential._helpers.capturing.canonical import ( + canonical_plan_capture, + canonical_value, + mask_console_text, + mask_text, +) +from scripts.planner_differential._helpers.capturing.state_tables import ( + state_delta, + state_rows, +) +from scripts.planner_differential._helpers.running.commands import harness_environment, run_sqb +from scripts.planner_differential._helpers.running.snapshot import ( + project_files, + restore_project_files, +) +from scripts.planner_differential._helpers.steps.arguments import step_arguments +from scripts.planner_differential._helpers.steps.mutations import apply_mutation +from scripts.planner_differential._helpers.steps.targets import plan_models, role_target +from scripts.planner_differential.constants import ( + BUILD_SUMMARY_PREFIXES, + CAPTURE_DIRECTORY, + COPY_IGNORED, + ERROR_LINE_PATTERN, + PLAN_CAPTURE_ENV_VAR, + PROJECT_DIRECTORY, + PROJECT_PATH_MASK, + WORK_PATH_MASK, +) +from scripts.planner_differential.models import ( + CommandResult, + ModelFacts, + MutationResult, + PlannerProject, + RunOptions, + StepContext, + TextMasks, +) + +_SOURCE_MAP_KEY: str = "source_map" +_SELECTION_FLAGS: frozenset[str] = frozenset({"--select", "--exclude"}) + + +def project_slug(name: str) -> str: + """Return a file name for a project name such as `fixture/orders`.""" + + return name.replace("/", "__") + + +def run_project(*, project: PlannerProject, options: RunOptions) -> dict[str, object]: + """Build the project once per step after its mutation and return the step records.""" + + case_dir: Path = options.work_dir / project_slug(project.name) + project_dir: Path = _prepared_project(project=project, case_dir=case_dir) + masks: TextMasks = TextMasks( + replacements=tuple( + (str(path), mask) + for path, mask in ( + (project_dir.resolve(), PROJECT_PATH_MASK), + (project_dir, PROJECT_PATH_MASK), + (options.work_dir.resolve(), WORK_PATH_MASK), + (options.work_dir, WORK_PATH_MASK), + ) + ), + today=date.today(), + ) + environment: dict[str, str] = harness_environment() + models: tuple[ModelFacts, ...] = () + source_tables: tuple[str, ...] = () + previous_state: dict[str, list[str]] = {} + records: list[dict[str, object]] = [] + for index, step in enumerate(options.steps): + context: StepContext = StepContext( + project_dir=project_dir, + models=models, + source_tables=source_tables, + environment=environment, + python=options.python, + ) + record: dict[str, object] = {"step": step.name} + records.append(record) + target: ModelFacts | None = ( + None + if step.role is None + else role_target( + role=step.role, index=step.index, models=models, project_dir=project_dir + ) + ) + if step.role is not None and target is None: + record["skipped"] = f"no {step.role} model" + continue + arguments: tuple[str, ...] | None = step_arguments( + step=step, context=context, target=target + ) + if arguments is None: + record["skipped"] = "no value for a flag placeholder" + continue + saved: dict[str, bytes] = project_files(project_dir) + if step.mutation is not None and target is not None: + mutation: MutationResult = apply_mutation( + name=step.mutation, context=context, target=target + ) + record["mutation"] = { + "applied": mutation.applied, + "note": mask_text(text=mutation.note, masks=masks), + } + record["arguments"] = list(arguments) + capture_dir: Path = case_dir / CAPTURE_DIRECTORY / f"{index:02d}-{step.name}" + shutil.rmtree(capture_dir, ignore_errors=True) + plan_json: CommandResult = run_sqb( + arguments=("plan", "--json", *arguments), + project_dir=project_dir, + environment={**environment, PLAN_CAPTURE_ENV_VAR: str(capture_dir)}, + python=options.python, + ) + record["plan_json"] = _plan_json_record(result=plan_json, masks=masks) + plan_output: list[object] = _plan_capture(capture_dir=capture_dir, masks=masks) + record["plan_output"] = plan_output + plan_text: CommandResult = run_sqb( + arguments=("plan", *arguments), + project_dir=project_dir, + environment=environment, + python=options.python, + ) + record["plan_text"] = { + "exit_code": plan_text.exit_code, + "lines": mask_console_text(text=plan_text.stdout, masks=masks).splitlines(), + "diagnostics": _diagnostics(result=plan_text, masks=masks), + } + build: CommandResult = run_sqb( + arguments=("build", *arguments), + project_dir=project_dir, + environment=environment, + python=options.python, + ) + record["build"] = _build_record(result=build, masks=masks) + state: dict[str, list[str]] = state_rows(project_dir=project_dir, masks=masks) + record["state"] = state_delta(before=previous_state, after=state) + previous_state = state + if plan_json.exit_code == 0 and not set(arguments) & _SELECTION_FLAGS: + models = plan_models(plan_json.stdout) + elif plan_json.exit_code != 0 and step.mutation is not None: + restore_project_files(project_dir=project_dir, files=saved) + record["reverted"] = True + source_tables = _source_tables(plan_output) or source_tables + if not options.keep_work: + shutil.rmtree(case_dir, ignore_errors=True) + return {"project": project.name, "steps": records} + + +def _prepared_project(*, project: PlannerProject, case_dir: Path) -> Path: + project_dir: Path = case_dir / PROJECT_DIRECTORY + shutil.rmtree(case_dir, ignore_errors=True) + if project.source_dir is not None: + _ = shutil.copytree( + project.source_dir, project_dir, ignore=shutil.ignore_patterns(*COPY_IGNORED) + ) + elif project.writer is not None: + project_dir.mkdir(parents=True) + project.writer(project_dir) + return project_dir + + +def _plan_json_record(*, result: CommandResult, masks: TextMasks) -> dict[str, object]: + try: + document: object = canonical_value(value=json.loads(result.stdout), masks=masks) + except json.JSONDecodeError: + document = mask_console_text(text=result.stdout, masks=masks).splitlines() + return { + "exit_code": result.exit_code, + "document": document, + "diagnostics": _diagnostics(result=result, masks=masks), + } + + +def _plan_capture(*, capture_dir: Path, masks: TextMasks) -> list[object]: + return [ + canonical_plan_capture(text=path.read_text(encoding="utf-8"), masks=masks) + for path in sorted(capture_dir.glob("*.json")) + ] + + +def _build_record(*, result: CommandResult, masks: TextMasks) -> dict[str, object]: + output: str = mask_console_text(text=result.stdout + result.stderr, masks=masks) + lines: list[str] = [line.strip() for line in output.splitlines() if line.strip()] + summary: list[str] = [line for line in lines if line.startswith(BUILD_SUMMARY_PREFIXES)] + return { + "exit_code": result.exit_code, + "summary": summary, + "diagnostics": sorted({line for line in lines if ERROR_LINE_PATTERN.match(line)}), + "last_line": lines[-1] if lines and result.exit_code != 0 and not summary else None, + } + + +def _diagnostics(*, result: CommandResult, masks: TextMasks) -> list[str]: + return [ + line.strip() + for line in mask_console_text(text=result.stderr, masks=masks).splitlines() + if ERROR_LINE_PATTERN.match(line) + ] + + +def _source_tables(plan_output: list[object]) -> tuple[str, ...]: + """Return the warehouse tables of the last captured plan's non-expression sources.""" + + last: dict[str, object] | None = as_json_object(plan_output[-1] if plan_output else None) + source_map: dict[str, object] | None = as_json_object( + None if last is None else last.get(_SOURCE_MAP_KEY) + ) + tables: list[str] = [] + for name, value in sorted((source_map or {}).items()): + entry: dict[str, object] | None = as_json_object(value) + if entry is None or entry.get("expression") is not None: + continue + table: str = str(entry.get("table") or name) + schema: object = entry.get("schema") + tables.append(table if schema is None else f"{schema}.{table}") + return tuple(tables) diff --git a/scripts/planner_differential/_helpers/running/report.py b/scripts/planner_differential/_helpers/running/report.py new file mode 100644 index 0000000000..9e1bd0af92 --- /dev/null +++ b/scripts/planner_differential/_helpers/running/report.py @@ -0,0 +1,38 @@ +"""Render planner differential results for terminals and CI logs.""" + +from __future__ import annotations + +from scripts.planner_differential.constants import MODE_RECORD +from scripts.planner_differential.models import ProjectDifference, ProjectResult + + +def format_result(result: ProjectResult) -> str: + """Render one project's verdict, harness error and every readable diff.""" + + status: str = "ERROR" if result.error else ("DIFF " if result.differences else "OK ") + lines: list[str] = [f"{status} {result.project} ({result.seconds:.1f}s)"] + if result.error: + lines.extend(f" {line}" for line in result.error.rstrip().splitlines()) + for difference in result.differences: + lines.append(_format_difference(difference)) + return "\n".join(lines) + + +def format_summary(*, results: list[ProjectResult], mode: str, seconds: float) -> str: + """Summarize the run; a difference or harness error reports FAILED.""" + + failed: list[ProjectResult] = [ + result for result in results if result.differences or result.error + ] + if not failed: + verb: str = "recorded" if mode == MODE_RECORD else "matched the baseline" + return f"Planner differential passed: {len(results)} projects {verb} in {seconds:.1f}s" + return ( + f"Planner differential FAILED: {len(failed)} of {len(results)} projects differ or " + f"errored in {seconds:.1f}s" + ) + + +def _format_difference(difference: ProjectDifference) -> str: + body: str = "\n".join(f" {line}" for line in difference.diff.splitlines()) + return f" - step {difference.step}: {difference.artifact}\n{body}" diff --git a/scripts/planner_differential/_helpers/running/run.py b/scripts/planner_differential/_helpers/running/run.py new file mode 100644 index 0000000000..7c891b83d4 --- /dev/null +++ b/scripts/planner_differential/_helpers/running/run.py @@ -0,0 +1,74 @@ +"""Run the corpus in parallel and compare or record each project.""" + +from __future__ import annotations + +import sys +import time +import traceback +from concurrent.futures import ThreadPoolExecutor + +from scripts.planner_differential._helpers.baseline.baseline import ( + compare_documents, + read_document, + write_document, +) +from scripts.planner_differential._helpers.running.project_run import project_slug, run_project +from scripts.planner_differential._helpers.running.report import format_result +from scripts.planner_differential.constants import ACTUAL_SUFFIX, MODE_RECORD +from scripts.planner_differential.models import PlannerProject, ProjectResult, RunOptions + + +def run_corpus(*, projects: list[PlannerProject], options: RunOptions) -> list[ProjectResult]: + """Run every project, printing each verdict as it completes.""" + + print( + f"Planner differential {options.mode}: {len(projects)} projects x " + f"{len(options.steps)} steps ({options.jobs} jobs), baseline {options.baseline_dir}", + file=sys.stderr, + ) + results: list[ProjectResult] = [] + with ThreadPoolExecutor(max_workers=options.jobs) as pool: + for result in pool.map( + lambda project: _project_result(project=project, options=options), projects + ): + results.append(result) + print(format_result(result)) + sys.stdout.flush() + return results + + +def _project_result(*, project: PlannerProject, options: RunOptions) -> ProjectResult: + started: float = time.monotonic() + slug: str = project_slug(project.name) + try: + document: dict[str, object] = run_project(project=project, options=options) + except Exception: # noqa: BLE001 - one project's harness failure must not hide the others + return ProjectResult( + project=project.name, + seconds=time.monotonic() - started, + differences=(), + error=traceback.format_exc(), + ) + if options.mode == MODE_RECORD: + _ = write_document(directory=options.baseline_dir, slug=slug, document=document) + return ProjectResult( + project=project.name, seconds=time.monotonic() - started, differences=() + ) + _ = write_document( + directory=options.baseline_dir.with_name(options.baseline_dir.name + ACTUAL_SUFFIX), + slug=slug, + document=document, + ) + expected: dict[str, object] | None = read_document(directory=options.baseline_dir, slug=slug) + if expected is None: + return ProjectResult( + project=project.name, + seconds=time.monotonic() - started, + differences=(), + error=f"no baseline for {project.name}; run with `record` first", + ) + return ProjectResult( + project=project.name, + seconds=time.monotonic() - started, + differences=compare_documents(expected=expected, actual=document), + ) diff --git a/scripts/planner_differential/_helpers/running/snapshot.py b/scripts/planner_differential/_helpers/running/snapshot.py new file mode 100644 index 0000000000..f7fa54e95a --- /dev/null +++ b/scripts/planner_differential/_helpers/running/snapshot.py @@ -0,0 +1,37 @@ +"""Save and restore a project's authored files around a mutation that breaks planning.""" + +from __future__ import annotations + +import fnmatch +from pathlib import Path + +from scripts.planner_differential.constants import COPY_IGNORED + + +def project_files(project_dir: Path) -> dict[str, bytes]: + """Return every authored file's bytes, keyed by its project-relative path.""" + + return { + path.relative_to(project_dir).as_posix(): path.read_bytes() + for path in sorted(project_dir.rglob("*")) + if path.is_file() and not _ignored(path.relative_to(project_dir)) + } + + +def restore_project_files(*, project_dir: Path, files: dict[str, bytes]) -> None: + """Rewrite the saved files and remove authored files added since they were saved.""" + + for relative in project_files(project_dir): + if relative not in files: + (project_dir / relative).unlink() + for relative, contents in files.items(): + path: Path = project_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_bytes(contents) + + +def _ignored(relative: Path) -> bool: + for part in relative.parts: + if any(fnmatch.fnmatch(part, pattern) for pattern in COPY_IGNORED): + return True + return False diff --git a/scripts/planner_differential/_helpers/steps/__init__.py b/scripts/planner_differential/_helpers/steps/__init__.py new file mode 100644 index 0000000000..cbc29d81a9 --- /dev/null +++ b/scripts/planner_differential/_helpers/steps/__init__.py @@ -0,0 +1 @@ +"""The mutation script applied to every project.""" diff --git a/scripts/planner_differential/_helpers/steps/arguments.py b/scripts/planner_differential/_helpers/steps/arguments.py new file mode 100644 index 0000000000..7f26204b59 --- /dev/null +++ b/scripts/planner_differential/_helpers/steps/arguments.py @@ -0,0 +1,57 @@ +"""Resolve a step's flag placeholders against its target model.""" + +from __future__ import annotations + +import duckdb + +from scripts.planner_differential._helpers.capturing.state_tables import database_files +from scripts.planner_differential._helpers.steps.relations import schema_and_name +from scripts.planner_differential.models import ModelFacts, PlannerStep, StepContext + +_MODEL_PLACEHOLDER: str = "{model}" +_START_PLACEHOLDER: str = "{start_cursor}" +_END_PLACEHOLDER: str = "{end_cursor}" + + +def step_arguments( + *, step: PlannerStep, context: StepContext, target: ModelFacts | None +) -> tuple[str, ...] | None: + """Return the step's plan and build flags, or None when a placeholder has no value.""" + + values: dict[str, str] = {} + if target is not None: + values[_MODEL_PLACEHOLDER] = target.name + if any(_START_PLACEHOLDER in argument for argument in step.arguments): + bounds: tuple[str, str] | None = ( + None if target is None else _cursor_bounds(context=context, target=target) + ) + if bounds is not None: + values[_START_PLACEHOLDER], values[_END_PLACEHOLDER] = bounds + resolved: list[str] = [] + for argument in step.arguments: + if argument.startswith("{") and argument not in values: + return None + resolved.append(values.get(argument, argument)) + return tuple(resolved) + + +def _cursor_bounds(*, context: StepContext, target: ModelFacts) -> tuple[str, str] | None: + """The minimum and maximum cursor values the target's relation holds.""" + + if target.cursor_column is None: + return None + schema, name = schema_and_name(target.qualified_name) + relation: str = f'"{schema}"."{name}"' + column: str = f'"{target.cursor_column}"' + for database in database_files(context.project_dir): + with duckdb.connect(str(database), read_only=True) as connection: + try: + found: tuple[object, ...] | None = connection.execute( + f"SELECT CAST(min({column}) AS VARCHAR), CAST(max({column}) AS VARCHAR) " + f"FROM {relation}" + ).fetchone() + except duckdb.CatalogException: + continue + if found is not None and found[0] is not None and found[1] is not None: + return (str(found[0]), str(found[1])) + return None diff --git a/scripts/planner_differential/_helpers/steps/model_file.py b/scripts/planner_differential/_helpers/steps/model_file.py new file mode 100644 index 0000000000..f233e20021 --- /dev/null +++ b/scripts/planner_differential/_helpers/steps/model_file.py @@ -0,0 +1,50 @@ +"""Edit a SQL model file's MODEL header and query as text.""" + +from __future__ import annotations + +import re + +from scripts.planner_differential.models import ModelFile + +_HEADER_START: re.Pattern[str] = re.compile(r"^\s*MODEL\s*\(", re.MULTILINE) +_HEADER_END: re.Pattern[str] = re.compile(r"^[ \t]*\)[ \t]*;[ \t]*$", re.MULTILINE) +_STATEMENT_SEPARATOR: str = ";" +_INLINE_HEADER: re.Pattern[str] = re.compile(r"\A\s*MODEL\s*\((?P[^;]*)\)\s*;") + + +def split_model_file(text: str) -> ModelFile | None: + """Split a model file into its MODEL header and one-statement query, or None.""" + + header: str = "" + start: re.Match[str] | None = _HEADER_START.match(text) + if start is not None: + end: re.Match[str] | None = _HEADER_END.search(text, start.end()) + inline: re.Match[str] | None = _INLINE_HEADER.match(text) + if end is not None: + header = text[: end.end()] + elif inline is not None: + header = inline.group(0) + else: + return None + query: str = text[len(header) :].strip().rstrip(";").strip() + if not query or _STATEMENT_SEPARATOR in query or _HEADER_START.match(query): + return None + return ModelFile(header=header, query=query) + + +def join_model_file(model_file: ModelFile) -> str: + """Return the file text for a header and query.""" + + prefix: str = f"{model_file.header}\n\n" if model_file.header else "" + return f"{prefix}{model_file.query}\n" + + +def add_header_property(*, header: str, line: str) -> str: + """Insert one `name value,` property at the start of the MODEL header.""" + + if not header: + return f"MODEL (\n {line}\n);" + start: re.Match[str] | None = _HEADER_START.match(header) + if start is None: + return header + return f"{header[: start.end()]}\n {line}{header[start.end() :]}" diff --git a/scripts/planner_differential/_helpers/steps/mutations.py b/scripts/planner_differential/_helpers/steps/mutations.py new file mode 100644 index 0000000000..742ed11f9e --- /dev/null +++ b/scripts/planner_differential/_helpers/steps/mutations.py @@ -0,0 +1,245 @@ +"""The project mutations a step applies before planning.""" + +from __future__ import annotations + +import re +from collections.abc import Callable +from pathlib import Path + +import duckdb + +from scripts.planner_differential._helpers.capturing.state_tables import database_files +from scripts.planner_differential._helpers.running.commands import run_sqb +from scripts.planner_differential._helpers.steps.model_file import ( + add_header_property, + join_model_file, + split_model_file, +) +from scripts.planner_differential._helpers.steps.relations import schema_and_name +from scripts.planner_differential.constants import ( + ADDED_COLUMN_NAME, + ADDED_MODEL_NAME, + ADDED_MODEL_PATH, + EDITED_DESCRIPTION, + MUTATION_ADD_MODEL, + MUTATION_CONFIG, + MUTATION_DECLARE_COLUMN, + MUTATION_DROP_RELATION, + MUTATION_EDIT_QUERY, + MUTATION_RENAME_COLUMN, + MUTATION_RENAME_MODEL, + MUTATION_SOURCE_GROWTH, + RENAMED_SUFFIX, + SOURCE_GROWTH_INTERVAL, +) +from scripts.planner_differential.models import ( + CommandResult, + ModelFacts, + ModelFile, + MutationResult, + StepContext, +) + +_TAGS_PATTERN: re.Pattern[str] = re.compile(r"^(\s*tags\s*\[)", re.MULTILINE) +_COLUMNS_PATTERN: re.Pattern[str] = re.compile(r"^(\s*columns\s*\()", re.MULTILINE) +_EDIT_ALIAS: str = "planner_differential_edit" +_TAG: str = "planner_differential" +_TIME_TYPES: tuple[str, ...] = ("TIMESTAMP", "DATE") +_COLUMNS_SQL: str = ( + "SELECT column_name, data_type FROM information_schema.columns " + "WHERE table_schema = ? AND table_name = ? ORDER BY ordinal_position" +) +_SOURCE_COLUMNS_SQL: str = ( + "SELECT c.table_schema, c.column_name, c.data_type FROM information_schema.columns c " + "JOIN information_schema.tables t " + "ON t.table_schema = c.table_schema AND t.table_name = c.table_name " + "WHERE t.table_type = 'BASE TABLE' AND c.table_name = ? " + "AND (CAST(? AS VARCHAR) IS NULL OR c.table_schema = ?) " + "ORDER BY c.table_schema, c.ordinal_position" +) +_RELATION_TYPE_SQL: str = ( + "SELECT table_type FROM information_schema.tables WHERE table_schema = ? AND table_name = ?" +) +_VIEW_TYPE: str = "VIEW" + + +def apply_mutation(*, name: str, context: StepContext, target: ModelFacts) -> MutationResult: + """Apply the named mutation to the target model or its project.""" + + mutations: dict[str, Callable[..., MutationResult]] = { + MUTATION_EDIT_QUERY: _edit_query, + MUTATION_CONFIG: _config, + MUTATION_DECLARE_COLUMN: _declare_column, + MUTATION_RENAME_COLUMN: _rename_column, + MUTATION_ADD_MODEL: _add_model, + MUTATION_RENAME_MODEL: _rename_model, + MUTATION_DROP_RELATION: _drop_relation, + MUTATION_SOURCE_GROWTH: _source_growth, + } + return mutations[name](context=context, target=target) + + +def _edit_query(*, context: StepContext, target: ModelFacts) -> MutationResult: + return _rewrite_model( + context=context, + target=target, + rewrite=lambda model_file: ModelFile( + header=model_file.header, + query=f"SELECT *\nFROM (\n{model_file.query}\n) AS {_EDIT_ALIAS}", + ), + ) + + +def _config(*, context: StepContext, target: ModelFacts) -> MutationResult: + def rewrite(model_file: ModelFile) -> ModelFile: + header: str = ( + _TAGS_PATTERN.sub(rf"\g<1>{_TAG}, ", model_file.header, count=1) + if _TAGS_PATTERN.search(model_file.header) + else add_header_property(header=model_file.header, line=f"tags [{_TAG}],") + ) + return ModelFile(header=header, query=model_file.query) + + return _rewrite_model(context=context, target=target, rewrite=rewrite) + + +def _declare_column(*, context: StepContext, target: ModelFacts) -> MutationResult: + declaration: str = f'{ADDED_COLUMN_NAME} (description "{EDITED_DESCRIPTION}"),' + columns: list[str] = [ + f'{_EDIT_ALIAS}."{column}"' + for column, _ in _relation_columns(context=context, target=target) + ] + if not columns: + return MutationResult(applied=False, note=f"no built relation for {target.name}") + + def rewrite(model_file: ModelFile) -> ModelFile: + header: str = ( + _COLUMNS_PATTERN.sub(rf"\g<1>\n {declaration}", model_file.header, count=1) + if _COLUMNS_PATTERN.search(model_file.header) + else add_header_property(header=model_file.header, line=f"columns ({declaration}),") + ) + return ModelFile( + header=header, + query=( + f"SELECT {', '.join(columns)}, CAST(1 AS INTEGER) AS {ADDED_COLUMN_NAME}\n" + f"FROM (\n{model_file.query}\n) AS {_EDIT_ALIAS}" + ), + ) + + return _rewrite_model(context=context, target=target, rewrite=rewrite) + + +def _rename_column(*, context: StepContext, target: ModelFacts) -> MutationResult: + path: Path = context.project_dir / target.relative_path + model_file: ModelFile | None = split_model_file(path.read_text(encoding="utf-8")) + header: str = "" if model_file is None else model_file.header + columns: list[str] = [ + column + for column, _ in _relation_columns(context=context, target=target) + if column != target.cursor_column + and column != ADDED_COLUMN_NAME + and re.search(rf"\b{re.escape(column)}\b", header) is None + ] + if not columns: + return MutationResult(applied=False, note=f"no renameable column in {target.name}") + column: str = columns[-1] + return _sqb_mutation( + context=context, + arguments=("rename", f"{target.name}.{column}", f"{column}{RENAMED_SUFFIX}"), + ) + + +def _add_model(*, context: StepContext, target: ModelFacts) -> MutationResult: + path: Path = context.project_dir / ADDED_MODEL_PATH + _ = path.write_text( + f'MODEL (\n materialized table,\n description "{EDITED_DESCRIPTION}",\n);\n\n' + f'SELECT *\nFROM __ref("{target.name}")\n', + encoding="utf-8", + newline="\n", + ) + return MutationResult(applied=True, note=f"added {ADDED_MODEL_NAME} reading {target.name}") + + +def _rename_model(*, context: StepContext, target: ModelFacts) -> MutationResult: + return _sqb_mutation( + context=context, arguments=("rename", target.name, f"{target.name}{RENAMED_SUFFIX}") + ) + + +def _drop_relation(*, context: StepContext, target: ModelFacts) -> MutationResult: + schema, name = schema_and_name(target.qualified_name) + for database in database_files(context.project_dir): + with duckdb.connect(str(database)) as connection: + found: tuple[object, ...] | None = connection.execute( + _RELATION_TYPE_SQL, [schema, name] + ).fetchone() + if found is None: + continue + kind: str = "VIEW" if found[0] == _VIEW_TYPE else "TABLE" + _ = connection.execute(f'DROP {kind} "{schema}"."{name}"') + return MutationResult(applied=True, note=f"dropped {kind.lower()} {schema}.{name}") + return MutationResult(applied=False, note=f"no relation {target.qualified_name}") + + +def _source_growth(*, context: StepContext, target: ModelFacts) -> MutationResult: + _ = target + grown: list[str] = [] + for source in context.source_tables: + schema: str | None = source.rpartition(".")[0] or None + name: str = source.rpartition(".")[2] + for database in database_files(context.project_dir): + with duckdb.connect(str(database)) as connection: + columns: list[tuple[str, str, str]] = connection.execute( + _SOURCE_COLUMNS_SQL, [name, schema, schema] + ).fetchall() + time_column: tuple[str, str, str] | None = next( + (column for column in columns if column[2].startswith(_TIME_TYPES)), None + ) + if time_column is None: + continue + table_schema, column, column_type = time_column + relation: str = f'"{table_schema}"."{name}"' + _ = connection.execute( + f'INSERT INTO {relation} SELECT * REPLACE (CAST("{column}" + ' + f'{SOURCE_GROWTH_INTERVAL} AS {column_type}) AS "{column}") ' + f'FROM {relation} WHERE "{column}" = (SELECT max("{column}") FROM {relation})' + ) + grown.append(f"{table_schema}.{name}.{column}") + if not grown: + return MutationResult(applied=False, note="no source table with a time column") + return MutationResult(applied=True, note=f"grew {', '.join(grown)}") + + +def _rewrite_model( + *, context: StepContext, target: ModelFacts, rewrite: Callable[[ModelFile], ModelFile] +) -> MutationResult: + path: Path = context.project_dir / target.relative_path + model_file: ModelFile | None = split_model_file(path.read_text(encoding="utf-8")) + if model_file is None: + return MutationResult(applied=False, note=f"cannot edit {target.relative_path}") + _ = path.write_text(join_model_file(rewrite(model_file)), encoding="utf-8", newline="\n") + return MutationResult(applied=True, note=f"edited {target.relative_path}") + + +def _sqb_mutation(*, context: StepContext, arguments: tuple[str, ...]) -> MutationResult: + result: CommandResult = run_sqb( + arguments=arguments, + project_dir=context.project_dir, + environment=context.environment, + python=context.python, + ) + return MutationResult( + applied=result.exit_code == 0, + note=f"sqb {' '.join(arguments)} exited {result.exit_code}", + ) + + +def _relation_columns(*, context: StepContext, target: ModelFacts) -> list[tuple[str, str]]: + schema, name = schema_and_name(target.qualified_name) + for database in database_files(context.project_dir): + with duckdb.connect(str(database), read_only=True) as connection: + columns: list[tuple[str, str]] = connection.execute( + _COLUMNS_SQL, [schema, name] + ).fetchall() + if columns: + return columns + return [] diff --git a/scripts/planner_differential/_helpers/steps/relations.py b/scripts/planner_differential/_helpers/steps/relations.py new file mode 100644 index 0000000000..aa72d48431 --- /dev/null +++ b/scripts/planner_differential/_helpers/steps/relations.py @@ -0,0 +1,12 @@ +"""Split relation names the plan reports.""" + +from __future__ import annotations + +_DEFAULT_SCHEMA: str = "main" + + +def schema_and_name(qualified_name: str) -> tuple[str, str]: + """Return a `schema.name` or `database.schema.name` relation's schema and name.""" + + parts: list[str] = qualified_name.split(".") + return (parts[-2] if len(parts) > 1 else _DEFAULT_SCHEMA, parts[-1]) diff --git a/scripts/planner_differential/_helpers/steps/targets.py b/scripts/planner_differential/_helpers/steps/targets.py new file mode 100644 index 0000000000..802783ee4c --- /dev/null +++ b/scripts/planner_differential/_helpers/steps/targets.py @@ -0,0 +1,107 @@ +"""Choose the model each step mutates or selects from the previous plan.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +from scripts.planner_differential._helpers.steps.model_file import split_model_file +from scripts.planner_differential.constants import ( + ADDED_MODEL_NAME, + ROLE_ANY, + ROLE_CURSOR, + ROLE_INCREMENTAL, + ROLE_LEAF, + ROLE_MICROBATCH, + ROLE_TABLE, +) +from scripts.planner_differential.models import ModelFacts + +_MATERIALIZATION_TABLE: str = "table" +_MATERIALIZATION_INCREMENTAL: str = "incremental" +_MICROBATCH_MODE: str = "microbatch" +_SQL_SUFFIX: str = ".sql" +_NAME_KEY: str = "name" +_TARGET_DIRECTORY: str = "target" +_SEARCHED_SUFFIXES: frozenset[str] = frozenset({".sql", ".yml", ".yaml", ".py", ".toml"}) + + +def plan_models(plan_json: str) -> tuple[ModelFacts, ...]: + """Return the models a `sqb plan --json` document lists, or none when it is not JSON.""" + + try: + document: object = json.loads(plan_json) + except json.JSONDecodeError: + return () + entries: object = document.get("models") if isinstance(document, dict) else None + if not isinstance(entries, list): + return () + return tuple( + ModelFacts( + name=str(entry[_NAME_KEY]), + relative_path=str(entry.get("relative_path") or ""), + materialization=str(entry.get("materialization_type") or ""), + qualified_name=str(entry.get("qualified_name") or ""), + cursor_column=entry.get("cursor_column"), + microbatch=entry.get("incremental_mode") == _MICROBATCH_MODE, + ) + for entry in entries + if isinstance(entry, dict) and _NAME_KEY in entry + ) + + +def role_target( + *, role: str, index: int, models: tuple[ModelFacts, ...], project_dir: Path +) -> ModelFacts | None: + """Return the role's `index`-th candidate (wrapping), or None when it has none.""" + + candidates: list[ModelFacts] = sorted( + ( + model + for model in models + if model.name != ADDED_MODEL_NAME + and _editable(model=model, project_dir=project_dir) + and _has_role(model=model, role=role, project_dir=project_dir) + ), + key=lambda model: model.name, + ) + return candidates[index % len(candidates)] if candidates else None + + +def _editable(*, model: ModelFacts, project_dir: Path) -> bool: + path: Path = project_dir / model.relative_path + return ( + model.relative_path.endswith(_SQL_SUFFIX) + and path.is_file() + and split_model_file(path.read_text(encoding="utf-8")) is not None + ) + + +def _has_role(*, model: ModelFacts, role: str, project_dir: Path) -> bool: + if role == ROLE_TABLE: + return model.materialization == _MATERIALIZATION_TABLE + if role == ROLE_INCREMENTAL: + return model.materialization == _MATERIALIZATION_INCREMENTAL + if role == ROLE_MICROBATCH: + return model.microbatch + if role == ROLE_CURSOR: + return model.cursor_column is not None + if role == ROLE_LEAF: + return model.materialization == _MATERIALIZATION_TABLE and not _referenced( + model=model, project_dir=project_dir + ) + return role == ROLE_ANY + + +def _referenced(*, model: ModelFacts, project_dir: Path) -> bool: + pattern: re.Pattern[str] = re.compile(rf"\b{re.escape(model.name)}\b") + own: Path = project_dir / model.relative_path + return any( + path != own + and path.suffix in _SEARCHED_SUFFIXES + and _TARGET_DIRECTORY not in path.relative_to(project_dir).parts + and pattern.search(path.read_text(encoding="utf-8", errors="replace")) is not None + for path in sorted(project_dir.rglob("*")) + if path.is_file() + ) diff --git a/scripts/planner_differential/constants.py b/scripts/planner_differential/constants.py new file mode 100644 index 0000000000..1d15d6fec6 --- /dev/null +++ b/scripts/planner_differential/constants.py @@ -0,0 +1,138 @@ +"""Constants for the planner differential.""" + +from __future__ import annotations + +import re + +from scripts.planner_differential.models import PlannerStep + +PLAN_CAPTURE_ENV_VAR: str = "SQLBUILD_PLANNER_CAPTURE_DIR" +MODE_RECORD: str = "record" +MODE_CHECK: str = "check" +MODES: tuple[str, ...] = (MODE_RECORD, MODE_CHECK) +DEFAULT_BASELINE_DIR: str = "/tmp/sqlbuild-planner-differential/baseline" +DEFAULT_SEED_COUNT: int = 6 +DEFAULT_JOBS: int = 2 +BASELINE_SUFFIX: str = ".json" +ACTUAL_SUFFIX: str = ".actual" +FIXTURE_ROOT: str = "tests/e2e/fixtures" +EXAMPLE_ROOT: str = "website/examples" +PROJECT_CONFIG_FILE: str = "sqlbuild_project.toml" +PROJECT_DIRECTORY: str = "project" +CAPTURE_DIRECTORY: str = "capture" +DUCKDB_ADAPTER: str = "duckdb" +DUCKDB_SUFFIX: str = ".duckdb" +COMMAND_TIMEOUT_SECONDS: float = 600.0 +SQB_ENTRY: str = "import sys; from sqlbuild.cli.entry.main.entry import main; sys.exit(main())" +EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset( + {"VIRTUAL_ENV", PLAN_CAPTURE_ENV_VAR, "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR"} +) +EXCLUDED_ENVIRONMENT_PREFIX: str = "DBT_" +COPY_IGNORED: tuple[str, ...] = ( + "target", + "logs", + "__pycache__", + ".sqlbuild", + "*.duckdb", + "*.duckdb.wal", +) + +STATE_TABLE_PREFIX: str = "_sqlbuild" +ADDED_MODEL_NAME: str = "planner_differential_added" +ADDED_MODEL_PATH: str = "models/planner_differential_added.sql" +ADDED_COLUMN_NAME: str = "planner_differential_flag" +RENAMED_SUFFIX: str = "_renamed" +EDITED_DESCRIPTION: str = "Edited by the planner differential." +SOURCE_GROWTH_INTERVAL: str = "INTERVAL 1 DAY" +MAX_DIFF_LINES: int = 60 + +ROLE_TABLE: str = "table" +ROLE_INCREMENTAL: str = "incremental" +ROLE_MICROBATCH: str = "microbatch" +ROLE_CURSOR: str = "cursor" +ROLE_LEAF: str = "leaf" +ROLE_ANY: str = "any" + +MUTATION_EDIT_QUERY: str = "edit_query" +MUTATION_CONFIG: str = "config" +MUTATION_DECLARE_COLUMN: str = "declare_column" +MUTATION_RENAME_COLUMN: str = "rename_column" +MUTATION_ADD_MODEL: str = "add_model" +MUTATION_RENAME_MODEL: str = "rename_model" +MUTATION_DROP_RELATION: str = "drop_relation" +MUTATION_SOURCE_GROWTH: str = "source_growth" + +SELECT_MODEL_ARGUMENTS: tuple[str, ...] = ("--select", "{model}") +STEPS: tuple[PlannerStep, ...] = ( + PlannerStep(name="initial"), + PlannerStep(name="no-op"), + PlannerStep(name="edit-query", mutation=MUTATION_EDIT_QUERY, role=ROLE_TABLE), + PlannerStep(name="config-change", mutation=MUTATION_CONFIG, role=ROLE_TABLE, index=1), + PlannerStep(name="declared-column", mutation=MUTATION_DECLARE_COLUMN, role=ROLE_INCREMENTAL), + PlannerStep(name="column-rename", mutation=MUTATION_RENAME_COLUMN, role=ROLE_INCREMENTAL), + PlannerStep(name="add-model", mutation=MUTATION_ADD_MODEL, role=ROLE_TABLE), + PlannerStep(name="rename-model", mutation=MUTATION_RENAME_MODEL, role=ROLE_LEAF), + PlannerStep(name="drop-relation", mutation=MUTATION_DROP_RELATION, role=ROLE_TABLE), + PlannerStep(name="source-growth", mutation=MUTATION_SOURCE_GROWTH, role=ROLE_CURSOR), + PlannerStep( + name="max-microbatches", role=ROLE_MICROBATCH, arguments=("--max-microbatches", "1") + ), + PlannerStep( + name="cursor-override", + role=ROLE_CURSOR, + arguments=( + *SELECT_MODEL_ARGUMENTS, + "--start-cursor", + "{start_cursor}", + "--end-cursor", + "{end_cursor}", + ), + ), + PlannerStep( + name="exclude-edited", + mutation=MUTATION_EDIT_QUERY, + role=ROLE_TABLE, + index=1, + arguments=("--exclude", "{model}"), + ), + PlannerStep(name="select-edited", role=ROLE_TABLE, index=1, arguments=SELECT_MODEL_ARGUMENTS), + PlannerStep(name="full-refresh", arguments=("--full-refresh",)), + PlannerStep(name="final-no-op"), +) + +PROJECT_PATH_MASK: str = "" +WORK_PATH_MASK: str = "" +RECENT_TIMESTAMP_MASK: str = "" +VOLATILE_VALUE_MASK: str = "" +ELAPSED_TIME_PATTERN: re.Pattern[str] = re.compile(r"\b\d+(?:\.\d+)?\s?m?s\b") +ELAPSED_TIME_MASK: str = "" +ISO_TIMESTAMP_PATTERN: re.Pattern[str] = re.compile( + r"\b(\d{4}-\d{2}-\d{2})[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:?\d{2}|Z)?" +) +COMPACT_TIMESTAMP_PATTERN: re.Pattern[str] = re.compile(r"(? int: + """Run the corpus; in check mode exit 1 when any project differs from its baseline.""" + + args: argparse.Namespace = parse_args(argv) + projects: list[PlannerProject] = filter_corpus( + projects=build_corpus( + repo_root=Path(__file__).resolve().parents[3], + seed_start=args.seed_start, + seeds=args.seeds, + ), + names=tuple(args.projects), + limit=args.limit, + ) + started: float = time.monotonic() + with tempfile.TemporaryDirectory(prefix="sqlbuild-planner-differential-") as temporary: + results: list[ProjectResult] = run_corpus( + projects=projects, + options=run_options(args=args, work_dir=args.work_dir or Path(temporary)), + ) + print(format_summary(results=results, mode=args.mode, seconds=time.monotonic() - started)) + return 1 if any(result.differences or result.error for result in results) else 0 diff --git a/scripts/planner_differential/models.py b/scripts/planner_differential/models.py new file mode 100644 index 0000000000..2a0d8704f4 --- /dev/null +++ b/scripts/planner_differential/models.py @@ -0,0 +1,116 @@ +"""Typed records for the planner differential.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from datetime import date +from pathlib import Path + + +@dataclass(frozen=True) +class PlannerProject: + """One project the mutation script runs on.""" + + name: str + source_dir: Path | None = None + writer: Callable[[Path], None] | None = None + + +@dataclass(frozen=True) +class PlannerStep: + """One mutation of a `role` model (candidate `index`), then plan and build with `arguments`.""" + + name: str + mutation: str | None = None + role: str | None = None + index: int = 0 + arguments: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ModelFacts: + """What the previous plan says about one model, for choosing step targets.""" + + name: str + relative_path: str + materialization: str + qualified_name: str + cursor_column: str | None + microbatch: bool + + +@dataclass(frozen=True) +class ModelFile: + """A SQL model file's MODEL header (possibly empty) and its single query.""" + + header: str + query: str + + +@dataclass(frozen=True) +class StepContext: + """Where a step runs and the models the previous plan reported.""" + + project_dir: Path + models: tuple[ModelFacts, ...] + source_tables: tuple[str, ...] + environment: dict[str, str] + python: Path + + +@dataclass(frozen=True) +class MutationResult: + """Whether a mutation applied, and a stable note describing what it changed.""" + + applied: bool + note: str + + +@dataclass(frozen=True) +class TextMasks: + """Run-specific text to mask: path prefixes by their mask, and the run's date.""" + + replacements: tuple[tuple[str, str], ...] + today: date + + +@dataclass(frozen=True) +class CommandResult: + """One sqb invocation's exit code and output.""" + + exit_code: int + stdout: str + stderr: str + + +@dataclass(frozen=True) +class RunOptions: + """Resolved harness settings.""" + + work_dir: Path + jobs: int + python: Path + mode: str + baseline_dir: Path + keep_work: bool + steps: tuple[PlannerStep, ...] + + +@dataclass(frozen=True) +class ProjectDifference: + """One artifact of one step that differs from the baseline, with a readable diff.""" + + step: str + artifact: str + diff: str + + +@dataclass(frozen=True) +class ProjectResult: + """A project's verdict against the baseline.""" + + project: str + seconds: float + differences: tuple[ProjectDifference, ...] + error: str | None = None diff --git a/scripts/run_planner_differential.py b/scripts/run_planner_differential.py new file mode 100644 index 0000000000..ad5996f687 --- /dev/null +++ b/scripts/run_planner_differential.py @@ -0,0 +1,15 @@ +"""Direct wrapper for the planner differential harness.""" + +from __future__ import annotations + +from scripts.planner_differential.main.differential import run_planner_differential + + +def main(argv: list[str] | None = None) -> int: + """Record or check stateful planner outputs of the corpus across a mutation script.""" + + return run_planner_differential(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/e2e/scripts/planner_differential/__init__.py b/tests/e2e/scripts/planner_differential/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/scripts/planner_differential/_test_types.py b/tests/e2e/scripts/planner_differential/_test_types.py new file mode 100644 index 0000000000..73026a02fa --- /dev/null +++ b/tests/e2e/scripts/planner_differential/_test_types.py @@ -0,0 +1,20 @@ +"""Test case types for the planner differential end-to-end tests.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class RecordCheckTestCase: + """A record run, a check of the same tree, and a check against a tampered baseline.""" + + description: str + project: str + steps: tuple[str, ...] + tampered_step_index: int + expected_artifacts: tuple[str, ...] + expected_clean_line: str + expected_difference_line: str + expected_diff_line: str + expected_note: str diff --git a/tests/e2e/scripts/planner_differential/helpers.py b/tests/e2e/scripts/planner_differential/helpers.py new file mode 100644 index 0000000000..cfa137836c --- /dev/null +++ b/tests/e2e/scripts/planner_differential/helpers.py @@ -0,0 +1,44 @@ +"""Harness arguments and baseline edits for the planner differential tests.""" + +from __future__ import annotations + +import json +from pathlib import Path + + +def harness_arguments( + *, mode: str, baseline_dir: Path, project: str, steps: tuple[str, ...] +) -> list[str]: + """Return arguments running one corpus project through some steps of the script.""" + + return [ + mode, + "--baseline-dir", + str(baseline_dir), + "--projects", + project, + "--seeds", + "0", + "--jobs", + "1", + "--steps", + *steps, + ] + + +def baseline_steps(path: Path) -> list[dict[str, object]]: + """Return a recorded project's step records.""" + + steps: object = json.loads(path.read_text(encoding="utf-8"))["steps"] + assert isinstance(steps, list) + return steps + + +def tamper_plan_exit_code(*, path: Path, step_index: int) -> None: + """Change one step's recorded `sqb plan --json` exit code, as a planner change would.""" + + document: dict[str, object] = json.loads(path.read_text(encoding="utf-8")) + records: object = document["steps"] + assert isinstance(records, list) + records[step_index]["plan_json"]["exit_code"] = 7 + _ = path.write_text(json.dumps(document), encoding="utf-8") diff --git a/tests/e2e/scripts/planner_differential/test_planner_differential.py b/tests/e2e/scripts/planner_differential/test_planner_differential.py new file mode 100644 index 0000000000..a5772cc81a --- /dev/null +++ b/tests/e2e/scripts/planner_differential/test_planner_differential.py @@ -0,0 +1,64 @@ +"""Record a stateful planner baseline, check the same tree, and catch a changed plan.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.planner_differential.main.differential import run_planner_differential +from tests.e2e.scripts.planner_differential._test_types import RecordCheckTestCase +from tests.e2e.scripts.planner_differential.helpers import ( + baseline_steps, + harness_arguments, + tamper_plan_exit_code, +) + + +@pytest.mark.parametrize( + "test_case", + [ + RecordCheckTestCase( + description="edited_model_on_a_built_project", + project="example/tidy-shop", + steps=("initial", "edit-query"), + tampered_step_index=1, + expected_artifacts=("plan_json", "plan_output", "plan_text", "build", "state"), + expected_clean_line="Planner differential passed: 1 projects matched the baseline", + expected_difference_line=" - step edit-query: plan_json", + expected_diff_line='+ "exit_code": 0,', + expected_note="edited models/daily_revenue.sql", + ), + ], + ids=lambda case: case.description, +) +def test_given_recorded_baseline_when_checking_then_only_a_changed_plan_differs( + test_case: RecordCheckTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + baseline_dir: Path = tmp_path / "baseline" + record_arguments: list[str] = harness_arguments( + mode="record", baseline_dir=baseline_dir, project=test_case.project, steps=test_case.steps + ) + check_arguments: list[str] = harness_arguments( + mode="check", baseline_dir=baseline_dir, project=test_case.project, steps=test_case.steps + ) + + assert run_planner_differential(record_arguments) == 0 + baseline: Path = baseline_dir / f"{test_case.project.replace('/', '__')}.json" + edited: dict[str, object] = baseline_steps(baseline)[-1] + assert edited["mutation"] == {"applied": True, "note": test_case.expected_note} + assert all(edited[artifact] for artifact in test_case.expected_artifacts) + _ = capsys.readouterr() + + assert run_planner_differential(check_arguments) == 0 + assert test_case.expected_clean_line in capsys.readouterr().out + + tamper_plan_exit_code(path=baseline, step_index=test_case.tampered_step_index) + assert run_planner_differential(check_arguments) == 1 + output: str = capsys.readouterr().out + assert test_case.expected_difference_line in output + assert test_case.expected_diff_line in output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/planner_differential/__init__.py b/tests/unit/scripts/planner_differential/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/planner_differential/_helpers/__init__.py b/tests/unit/scripts/planner_differential/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/planner_differential/_helpers/capturing/__init__.py b/tests/unit/scripts/planner_differential/_helpers/capturing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/planner_differential/_helpers/capturing/_test_types.py b/tests/unit/scripts/planner_differential/_helpers/capturing/_test_types.py new file mode 100644 index 0000000000..0569f7c71f --- /dev/null +++ b/tests/unit/scripts/planner_differential/_helpers/capturing/_test_types.py @@ -0,0 +1,42 @@ +"""Test case types for the planner differential canonicalisation helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class MaskTextTestCase: + """Raw output text and the text after masking run-specific values.""" + + description: str + raw: str + expected_text: str + + +@dataclass(frozen=True) +class CanonicalValueTestCase: + """A decoded JSON value and its canonical form.""" + + description: str + value: object + expected_value: object + + +@dataclass(frozen=True) +class StateDeltaTestCase: + """State rows before and after a build and the rows the step gained and lost.""" + + description: str + before: dict[str, list[str]] + after: dict[str, list[str]] + expected_delta: dict[str, dict[str, list[object]]] + + +@dataclass(frozen=True) +class PlanCaptureTestCase: + """A value written as a plan capture document and its canonical expanded form.""" + + description: str + value: object + expected_value: object diff --git a/tests/unit/scripts/planner_differential/_helpers/capturing/helpers.py b/tests/unit/scripts/planner_differential/_helpers/capturing/helpers.py new file mode 100644 index 0000000000..d96fa13bad --- /dev/null +++ b/tests/unit/scripts/planner_differential/_helpers/capturing/helpers.py @@ -0,0 +1,59 @@ +"""Masks and plan-like values for the canonicalisation tests.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, date, datetime + +from scripts.planner_differential.models import TextMasks + +RUN_DATE: date = date(2026, 10, 14) +TEST_MASKS: TextMasks = TextMasks( + replacements=( + ("/runs/work/orders/project", ""), + ("/runs/work", ""), + ), + today=RUN_DATE, +) +LONG_QUERY: str = "SELECT order_id FROM orders WHERE status = 'shipped'\n" * 30 + + +@dataclass(frozen=True) +class OrderPlanEntry: + """A plan entry shaped like the planner's: a name, a query and an invocation time.""" + + name: str + query: str + invocation_time: datetime + data_start: datetime + + +@dataclass(frozen=True) +class OrderPlan: + """A plan holding the same large entry twice, so the capture shares it.""" + + entries: tuple[OrderPlanEntry, ...] + + +def order_plan() -> OrderPlan: + """Return a plan whose entries repeat one large query and carry a volatile time.""" + + entry: OrderPlanEntry = OrderPlanEntry( + name="order_totals", + query=LONG_QUERY, + invocation_time=datetime(2026, 10, 14, 9, 30, tzinfo=UTC), + data_start=datetime(2026, 4, 1, tzinfo=UTC), + ) + return OrderPlan(entries=(entry, entry)) + + +def expected_order_entry() -> dict[str, object]: + """Return the canonical form of one `order_plan()` entry.""" + + return { + "__type__": f"{__name__}:OrderPlanEntry", + "name": "order_totals", + "query": LONG_QUERY, + "invocation_time": "", + "data_start": {"__datetime__": "2026-04-01T00:00:00+00:00"}, + } diff --git a/tests/unit/scripts/planner_differential/_helpers/capturing/test_canonical.py b/tests/unit/scripts/planner_differential/_helpers/capturing/test_canonical.py new file mode 100644 index 0000000000..b0431ca765 --- /dev/null +++ b/tests/unit/scripts/planner_differential/_helpers/capturing/test_canonical.py @@ -0,0 +1,206 @@ +"""Canonicalisation masks only run-specific values so two runs of one tree compare equal.""" + +from __future__ import annotations + +import pytest + +from scripts.planner_differential._helpers.capturing.canonical import ( + canonical_plan_capture, + canonical_value, + mask_console_text, + mask_text, +) +from scripts.planner_differential._helpers.capturing.state_tables import state_delta +from sqlbuild._native import render_stage_capture +from tests.unit.scripts.planner_differential._helpers.capturing._test_types import ( + CanonicalValueTestCase, + MaskTextTestCase, + PlanCaptureTestCase, + StateDeltaTestCase, +) +from tests.unit.scripts.planner_differential._helpers.capturing.helpers import ( + TEST_MASKS, + expected_order_entry, + order_plan, +) + + +@pytest.mark.parametrize( + "test_case", + [ + MaskTextTestCase( + description="run_id", + raw="run 20261014T002537Z_7e9b74a9fefc", + expected_text="run ", + ), + MaskTextTestCase( + description="project_path_before_work_path", + raw="/runs/work/orders/project/models/a.sql and /runs/work/other", + expected_text="/models/a.sql and /other", + ), + MaskTextTestCase( + description="timestamp_on_the_run_date", + raw="built at 2026-10-14T09:30:00.123+00:00", + expected_text="built at ", + ), + MaskTextTestCase( + description="retention_expiry_after_the_run_date", + raw="expires 2026-11-13 00:38:45.472672", + expected_text="expires ", + ), + MaskTextTestCase( + description="data_timestamp_is_kept", + raw="cursor 2026-04-02 12:00:00 to 2026-04-05 14:30:00", + expected_text="cursor 2026-04-02 12:00:00 to 2026-04-05 14:30:00", + ), + MaskTextTestCase( + description="compact_archive_timestamp", + raw="_sqb_archive__20261014t004426z__migration_origin__orders", + expected_text="_sqb_archive____migration_origin__orders", + ), + ], + ids=lambda case: case.description, +) +def test_given_output_text_when_masking_then_only_run_specific_values_change( + test_case: MaskTextTestCase, +) -> None: + assert mask_text(text=test_case.raw, masks=TEST_MASKS) == test_case.expected_text + + +@pytest.mark.parametrize( + "test_case", + [ + MaskTextTestCase( + description="elapsed_time", + raw="Generated plan. (0.04s)", + expected_text="Generated plan. ()", + ), + MaskTextTestCase( + description="project_path_in_a_diagnostic", + raw="error: /runs/work/orders/project/models/orders.sql:3:1", + expected_text="error: /models/orders.sql:3:1", + ), + ], + ids=lambda case: case.description, +) +def test_given_console_text_when_masking_then_elapsed_times_are_masked( + test_case: MaskTextTestCase, +) -> None: + assert mask_console_text(text=test_case.raw, masks=TEST_MASKS) == test_case.expected_text + + +@pytest.mark.parametrize( + "test_case", + [ + CanonicalValueTestCase( + description="volatile_keys", + value={"run_id": "r1", "ts": "t", "migration_event_id": "e", "name": "orders"}, + expected_value={ + "run_id": "", + "ts": "", + "migration_event_id": "", + "name": "orders", + }, + ), + CanonicalValueTestCase( + description="absent_volatile_value_stays_null", + value={"completed_at": None}, + expected_value={"completed_at": None}, + ), + CanonicalValueTestCase( + description="non_string_key_mapping_is_sorted", + value={"__mapping__": [[["model", "orders"], 2], [["loader", "customers"], 1]]}, + expected_value={ + "__mapping__": [[["loader", "customers"], 1], [["model", "orders"], 2]] + }, + ), + CanonicalValueTestCase( + description="list_order_is_kept", + value={"execution_order": ["orders", "customers"]}, + expected_value={"execution_order": ["orders", "customers"]}, + ), + CanonicalValueTestCase( + description="nested_strings_are_masked", + value=[{"path": "/runs/work/orders/project/models/orders.sql"}], + expected_value=[{"path": "/models/orders.sql"}], + ), + ], + ids=lambda case: case.description, +) +def test_given_decoded_json_when_canonicalising_then_volatile_values_are_masked( + test_case: CanonicalValueTestCase, +) -> None: + assert canonical_value(value=test_case.value, masks=TEST_MASKS) == test_case.expected_value + + +@pytest.mark.parametrize( + "test_case", + [ + StateDeltaTestCase( + description="appended_rows_are_added", + before={"db:main._sqlbuild_fingerprints": ['{"name": "orders"}']}, + after={"db:main._sqlbuild_fingerprints": ['{"name": "orders"}', '{"name": "orders"}']}, + expected_delta={ + "db:main._sqlbuild_fingerprints": {"added": [{"name": "orders"}], "removed": []} + }, + ), + StateDeltaTestCase( + description="replaced_and_new_tables", + before={"db:main._sqlbuild_migrations": ['{"state": "old"}']}, + after={ + "db:main._sqlbuild_migrations": ['{"state": "new"}'], + "db:main._sqlbuild_old_name_views": ['{"view": "orders"}'], + }, + expected_delta={ + "db:main._sqlbuild_migrations": { + "added": [{"state": "new"}], + "removed": [{"state": "old"}], + }, + "db:main._sqlbuild_old_name_views": { + "added": [{"view": "orders"}], + "removed": [], + }, + }, + ), + StateDeltaTestCase( + description="unchanged_tables_are_omitted", + before={"db:main._sqlbuild_fingerprints": ['{"name": "orders"}']}, + after={"db:main._sqlbuild_fingerprints": ['{"name": "orders"}']}, + expected_delta={}, + ), + ], + ids=lambda case: case.description, +) +def test_given_state_rows_when_diffing_then_gained_and_lost_rows_are_reported( + test_case: StateDeltaTestCase, +) -> None: + assert state_delta(before=test_case.before, after=test_case.after) == test_case.expected_delta + + +@pytest.mark.parametrize( + "test_case", + [ + PlanCaptureTestCase( + description="shared_entries_expanded_and_invocation_time_masked", + value=order_plan(), + expected_value={ + "__type__": ( + "tests.unit.scripts.planner_differential._helpers.capturing.helpers:OrderPlan" + ), + "entries": [expected_order_entry(), expected_order_entry()], + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_plan_capture_document_when_canonicalising_then_nodes_expand_and_mask( + test_case: PlanCaptureTestCase, +) -> None: + assert ( + canonical_plan_capture(text=render_stage_capture(test_case.value), masks=TEST_MASKS) + == test_case.expected_value + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/planner_differential/_helpers/steps/__init__.py b/tests/unit/scripts/planner_differential/_helpers/steps/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/planner_differential/_helpers/steps/_test_types.py b/tests/unit/scripts/planner_differential/_helpers/steps/_test_types.py new file mode 100644 index 0000000000..d2b5436449 --- /dev/null +++ b/tests/unit/scripts/planner_differential/_helpers/steps/_test_types.py @@ -0,0 +1,35 @@ +"""Test case types for the planner differential model-file edits.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from scripts.planner_differential.models import ModelFile + + +@dataclass(frozen=True) +class SplitModelFileTestCase: + """Model file text and the header and query the harness edits, or None.""" + + description: str + text: str + expected_model_file: ModelFile | None + + +@dataclass(frozen=True) +class HeaderPropertyTestCase: + """A MODEL header and the header after inserting one property line.""" + + description: str + header: str + line: str + expected_header: str + + +@dataclass(frozen=True) +class JoinModelFileTestCase: + """A split model file and the text the harness writes back.""" + + description: str + model_file: ModelFile + expected_text: str diff --git a/tests/unit/scripts/planner_differential/_helpers/steps/test_model_file.py b/tests/unit/scripts/planner_differential/_helpers/steps/test_model_file.py new file mode 100644 index 0000000000..7832794c03 --- /dev/null +++ b/tests/unit/scripts/planner_differential/_helpers/steps/test_model_file.py @@ -0,0 +1,108 @@ +"""Model-file edits split the MODEL header from a single query and insert properties.""" + +from __future__ import annotations + +import pytest + +from scripts.planner_differential._helpers.steps.model_file import ( + add_header_property, + join_model_file, + split_model_file, +) +from scripts.planner_differential.models import ModelFile +from tests.unit.scripts.planner_differential._helpers.steps._test_types import ( + HeaderPropertyTestCase, + JoinModelFileTestCase, + SplitModelFileTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SplitModelFileTestCase( + description="multi_line_header", + text="MODEL (\n materialized table,\n columns (id (nullable false)),\n);\n\n" + 'SELECT id FROM __ref("orders");\n', + expected_model_file=ModelFile( + header="MODEL (\n materialized table,\n columns (id (nullable false)),\n);", + query='SELECT id FROM __ref("orders")', + ), + ), + SplitModelFileTestCase( + description="inline_header", + text="MODEL (materialized view);\nSELECT 1 AS id\n", + expected_model_file=ModelFile( + header="MODEL (materialized view);", query="SELECT 1 AS id" + ), + ), + SplitModelFileTestCase( + description="no_header", + text="SELECT 1 AS id\n", + expected_model_file=ModelFile(header="", query="SELECT 1 AS id"), + ), + SplitModelFileTestCase( + description="several_statements_are_not_edited", + text="MODEL (\n materialized table,\n);\n\nSET x = 1;\nSELECT 1 AS id\n", + expected_model_file=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_model_file_when_splitting_then_header_and_query_are_separated( + test_case: SplitModelFileTestCase, +) -> None: + assert split_model_file(test_case.text) == test_case.expected_model_file + + +@pytest.mark.parametrize( + "test_case", + [ + HeaderPropertyTestCase( + description="existing_header", + header="MODEL (\n materialized table,\n);", + line="tags [orders],", + expected_header="MODEL (\n tags [orders],\n materialized table,\n);", + ), + HeaderPropertyTestCase( + description="no_header", + header="", + line="tags [orders],", + expected_header="MODEL (\n tags [orders],\n);", + ), + ], + ids=lambda case: case.description, +) +def test_given_header_when_adding_property_then_it_leads_the_header( + test_case: HeaderPropertyTestCase, +) -> None: + assert ( + add_header_property(header=test_case.header, line=test_case.line) + == test_case.expected_header + ) + + +@pytest.mark.parametrize( + "test_case", + [ + JoinModelFileTestCase( + description="header_and_query", + model_file=ModelFile(header="MODEL (\n materialized view,\n);", query="SELECT 1"), + expected_text="MODEL (\n materialized view,\n);\n\nSELECT 1\n", + ), + JoinModelFileTestCase( + description="query_only", + model_file=ModelFile(header="", query="SELECT 1"), + expected_text="SELECT 1\n", + ), + ], + ids=lambda case: case.description, +) +def test_given_split_file_when_joining_then_header_and_query_are_separated( + test_case: JoinModelFileTestCase, +) -> None: + assert join_model_file(test_case.model_file) == test_case.expected_text + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 1325534c6557c65c4eaf01c2463573fec5a994ba Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 03:35:45 +0100 Subject: [PATCH 05/40] test: cover config and materialization changes in the planner differential --- scripts/planner_differential/README.md | 11 +++-- .../_helpers/steps/mutations.py | 48 ++++++++++++++++++- scripts/planner_differential/constants.py | 8 +++- 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/scripts/planner_differential/README.md b/scripts/planner_differential/README.md index 582df4769f..22586276f3 100644 --- a/scripts/planner_differential/README.md +++ b/scripts/planner_differential/README.md @@ -7,7 +7,7 @@ that can alter planning (for example a port of planner code): record a baseline before the change, then check the changed tree against it. ```bash -# On the tree before the change (about 9 minutes with the defaults) +# On the tree before the change (about 10 minutes with the defaults) make planner-differential PLANNER_DIFFERENTIAL_MODE=record # On the changed tree make planner-differential @@ -42,10 +42,11 @@ relation. A step whose role has no model, or whose placeholders have no value, i skipped. Targets come from the previous unselected `sqb plan --json`, so later steps follow earlier renames. -The script covers a no-op rebuild, a query edit, a config-only change, a new declared column, a -column rename and a model rename through `sqb rename` (`migrate_from`), an added model, a -relation dropped outside SQLBuild, source rows added with a later cursor value, and the -`--max-microbatches`, cursor override, `--exclude`, `--select` and `--full-refresh` flags. To +The script covers a no-op rebuild, a query edit, a tag change (metadata only), a config change +(`on_schema_change`), a new declared column, a column rename and a model rename through +`sqb rename` (`migrate_from`), an added model, a relation dropped outside SQLBuild, source rows +added with a later cursor value, the `--max-microbatches`, cursor override, `--exclude` and +`--select` flags, a table-to-view materialization change, and `--full-refresh`. To add a step, append a `PlannerStep`; a new kind of change adds a mutation function in `_helpers/steps/mutations.py`. When a mutation makes planning fail, the step is recorded and the project files are restored so later steps still run. `--steps` runs a subset (include `initial` diff --git a/scripts/planner_differential/_helpers/steps/mutations.py b/scripts/planner_differential/_helpers/steps/mutations.py index 742ed11f9e..eeee8094c7 100644 --- a/scripts/planner_differential/_helpers/steps/mutations.py +++ b/scripts/planner_differential/_helpers/steps/mutations.py @@ -26,9 +26,11 @@ MUTATION_DECLARE_COLUMN, MUTATION_DROP_RELATION, MUTATION_EDIT_QUERY, + MUTATION_MATERIALIZATION, MUTATION_RENAME_COLUMN, MUTATION_RENAME_MODEL, MUTATION_SOURCE_GROWTH, + MUTATION_TAGS, RENAMED_SUFFIX, SOURCE_GROWTH_INTERVAL, ) @@ -41,6 +43,16 @@ ) _TAGS_PATTERN: re.Pattern[str] = re.compile(r"^(\s*tags\s*\[)", re.MULTILINE) +_SCHEMA_CHANGE_KEY: str = "on_schema_change" +_SCHEMA_CHANGE_PATTERN: re.Pattern[str] = re.compile( + rf"^(\s*{_SCHEMA_CHANGE_KEY}\s+)(\w+)", re.MULTILINE +) +_APPEND_NEW_COLUMNS: str = "append_new_columns" +_SCHEMA_CHANGE_TOGGLE: dict[str, str] = { + _APPEND_NEW_COLUMNS: "sync_all_columns", + "sync_all_columns": _APPEND_NEW_COLUMNS, +} +_MATERIALIZED_PATTERN: re.Pattern[str] = re.compile(r"^(\s*materialized\s+)\w+", re.MULTILINE) _COLUMNS_PATTERN: re.Pattern[str] = re.compile(r"^(\s*columns\s*\()", re.MULTILINE) _EDIT_ALIAS: str = "planner_differential_edit" _TAG: str = "planner_differential" @@ -68,7 +80,9 @@ def apply_mutation(*, name: str, context: StepContext, target: ModelFacts) -> Mu mutations: dict[str, Callable[..., MutationResult]] = { MUTATION_EDIT_QUERY: _edit_query, + MUTATION_TAGS: _tags, MUTATION_CONFIG: _config, + MUTATION_MATERIALIZATION: _materialization, MUTATION_DECLARE_COLUMN: _declare_column, MUTATION_RENAME_COLUMN: _rename_column, MUTATION_ADD_MODEL: _add_model, @@ -90,7 +104,7 @@ def _edit_query(*, context: StepContext, target: ModelFacts) -> MutationResult: ) -def _config(*, context: StepContext, target: ModelFacts) -> MutationResult: +def _tags(*, context: StepContext, target: ModelFacts) -> MutationResult: def rewrite(model_file: ModelFile) -> ModelFile: header: str = ( _TAGS_PATTERN.sub(rf"\g<1>{_TAG}, ", model_file.header, count=1) @@ -102,6 +116,38 @@ def rewrite(model_file: ModelFile) -> ModelFile: return _rewrite_model(context=context, target=target, rewrite=rewrite) +def _config(*, context: StepContext, target: ModelFacts) -> MutationResult: + def rewrite(model_file: ModelFile) -> ModelFile: + found: re.Match[str] | None = _SCHEMA_CHANGE_PATTERN.search(model_file.header) + value: str = ( + _SCHEMA_CHANGE_TOGGLE.get(found.group(2), _APPEND_NEW_COLUMNS) + if found is not None + else _APPEND_NEW_COLUMNS + ) + header: str = ( + _SCHEMA_CHANGE_PATTERN.sub(rf"\g<1>{value}", model_file.header, count=1) + if found is not None + else add_header_property( + header=model_file.header, line=f"{_SCHEMA_CHANGE_KEY} {value}," + ) + ) + return ModelFile(header=header, query=model_file.query) + + return _rewrite_model(context=context, target=target, rewrite=rewrite) + + +def _materialization(*, context: StepContext, target: ModelFacts) -> MutationResult: + def rewrite(model_file: ModelFile) -> ModelFile: + header: str = ( + _MATERIALIZED_PATTERN.sub(r"\g<1>view", model_file.header, count=1) + if _MATERIALIZED_PATTERN.search(model_file.header) + else add_header_property(header=model_file.header, line="materialized view,") + ) + return ModelFile(header=header, query=model_file.query) + + return _rewrite_model(context=context, target=target, rewrite=rewrite) + + def _declare_column(*, context: StepContext, target: ModelFacts) -> MutationResult: declaration: str = f'{ADDED_COLUMN_NAME} (description "{EDITED_DESCRIPTION}"),' columns: list[str] = [ diff --git a/scripts/planner_differential/constants.py b/scripts/planner_differential/constants.py index 1d15d6fec6..2ee124ef40 100644 --- a/scripts/planner_differential/constants.py +++ b/scripts/planner_differential/constants.py @@ -54,7 +54,9 @@ ROLE_ANY: str = "any" MUTATION_EDIT_QUERY: str = "edit_query" +MUTATION_TAGS: str = "tags" MUTATION_CONFIG: str = "config" +MUTATION_MATERIALIZATION: str = "materialization" MUTATION_DECLARE_COLUMN: str = "declare_column" MUTATION_RENAME_COLUMN: str = "rename_column" MUTATION_ADD_MODEL: str = "add_model" @@ -67,7 +69,8 @@ PlannerStep(name="initial"), PlannerStep(name="no-op"), PlannerStep(name="edit-query", mutation=MUTATION_EDIT_QUERY, role=ROLE_TABLE), - PlannerStep(name="config-change", mutation=MUTATION_CONFIG, role=ROLE_TABLE, index=1), + PlannerStep(name="tag-change", mutation=MUTATION_TAGS, role=ROLE_TABLE, index=1), + PlannerStep(name="config-change", mutation=MUTATION_CONFIG, role=ROLE_INCREMENTAL, index=1), PlannerStep(name="declared-column", mutation=MUTATION_DECLARE_COLUMN, role=ROLE_INCREMENTAL), PlannerStep(name="column-rename", mutation=MUTATION_RENAME_COLUMN, role=ROLE_INCREMENTAL), PlannerStep(name="add-model", mutation=MUTATION_ADD_MODEL, role=ROLE_TABLE), @@ -96,6 +99,9 @@ arguments=("--exclude", "{model}"), ), PlannerStep(name="select-edited", role=ROLE_TABLE, index=1, arguments=SELECT_MODEL_ARGUMENTS), + PlannerStep( + name="materialization-change", mutation=MUTATION_MATERIALIZATION, role=ROLE_TABLE, index=2 + ), PlannerStep(name="full-refresh", arguments=("--full-refresh",)), PlannerStep(name="final-no-op"), ) From 3008a31d0631543650ce6be9b25080866f956407 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 03:17:40 +0100 Subject: [PATCH 06/40] refactor: validate and render cursor intrinsics natively Callers use the native resolve_cursor_intrinsics, validated_model_cursor_intrinsics and cursor_intrinsics_analysis_sql bindings; the Rust glue calls the shared helpers directly and owns the reserved executor cursor markers. --- .../analysis_session/model_analyses.rs | 19 +- .../_helpers/attachments/cursor_intrinsics.rs | 183 +++++++++++++----- .../compile_attachment/model_inputs.rs | 4 +- .../compile_driver/assembly_checks.rs | 25 +-- .../compile_driver/assembly_models.rs | 28 +-- .../project_assembly/syntax_checks.rs | 18 +- .../semantic_shapes/source_bindings.rs | 31 ++- src/sqlbuild/_native.pyi | 13 +- .../_helpers/render/cursor_intrinsics.py | 70 ------- .../compile/main/cursor_intrinsics.py | 20 -- .../_helpers/resolve/cursor_intrinsics.py | 2 +- .../planner/_helpers/resolve/resolve.py | 2 +- .../_helpers/sql_tests/cursor_window.py | 3 +- .../executor/build/_helpers/scheduler.py | 3 +- .../run/_helpers/validation/cursor_bounds.py | 2 +- .../compiler/analysis_session/helpers.py | 4 +- .../_helpers/render/test_cursor_intrinsics.py | 18 +- 17 files changed, 203 insertions(+), 242 deletions(-) delete mode 100644 src/sqlbuild/compiler/compile/_helpers/render/cursor_intrinsics.py delete mode 100644 src/sqlbuild/compiler/compile/main/cursor_intrinsics.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 5fb7a0b6f9..7dea30fd8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -9,13 +9,12 @@ use crate::bindings::_helpers::analysis_session::profile_inputs::{ case_sensitive_shapes, nullability_rules, profile_dialect, return_types, shape_rows, }; use crate::bindings::_helpers::analysis_session::stage_functions::family_rows; +use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::boundary::keyword_request::Request; const NATIVE_MODULE: &str = "sqlbuild._native"; const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; const ANALYSIS_CACHE_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.cache"; -const CURSOR_INTRINSICS_MODULE: &str = - "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const CODE_IDENTITY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiled_code_identity"; @@ -277,9 +276,6 @@ fn model_rows<'py>( py: Python<'py>, model_inputs: &[Bound<'py, PyAny>], ) -> PyResult> { - let cursor_sql: Bound<'py, PyAny> = py - .import(CURSOR_INTRINSICS_MODULE)? - .getattr("cursor_intrinsics_analysis_sql")?; let planner = py.import(PLANNER_TYPES_MODULE)?; let enforced: Bound<'py, PyAny> = planner.getattr("ContractPolicy")?.getattr("ENFORCED")?; let snapshot: Bound<'py, PyAny> = planner @@ -289,10 +285,15 @@ fn model_rows<'py>( for model_input in model_inputs { let values: Bound<'py, PyAny> = model_input.getattr("config")?.getattr("values")?; let get = |key: &str| values.call_method1("get", (key,)); - let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("sql", model_input.getattr("query_sql")?)?; - arguments.set_item("cursor_type", get("cursor_type")?)?; - let query_sql: Bound<'py, PyAny> = cursor_sql.call((), Some(&arguments))?; + let query_sql: Bound<'py, PyAny> = PyString::new( + py, + &analysis_cursor_sql( + py, + &model_input.getattr("query_sql")?.extract::()?, + &get("cursor_type")?, + )?, + ) + .into_any(); let mut placeholders: Vec<(String, String)> = Vec::new(); let raw: Bound<'py, PyAny> = get("placeholders")?; if raw.is_instance_of::() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs index b36dcc7f9d..e942a97baf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs @@ -1,76 +1,157 @@ //! Cursor intrinsic validation, rejection and rendering. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{PyString, PyStringMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::{Python, pyfunction, wrap_pyfunction}; use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; use sqlbuild_attachments::cursor_intrinsics::main::validated_model_intrinsics::validated_model_intrinsics; use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::compile_driver::python_calls::imported; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const INCREMENTAL_MATERIALIZATION: &str = "incremental"; const CURSOR_START: &str = "__cursor_start"; +const CURSOR_START_CALL: &str = "__cursor_start()"; +const CURSOR_END_CALL: &str = "__cursor_end()"; +const INTEGER_CURSOR_TYPE: &str = "integer"; +const INTEGER_ANALYSIS_BOUND: &str = "CAST(0 AS BIGINT)"; +const TIMESTAMP_ANALYSIS_BOUND: &str = "CAST('2000-01-01 00:00:00' AS TIMESTAMP)"; +const MODEL_SQL_CONTEXT: &str = "Model SQL"; +const SQL_CONTEXT: &str = "SQL"; +const RESERVED_CURSOR_MARKERS: [&str; 2] = ["__SQB_CURSOR_START__", "__SQB_CURSOR_END__"]; -/// A model query with canonical intrinsic calls and no error, or no query and the error. -#[pyfunction] -fn validated_model_cursor_intrinsics( - sql: &str, - reserved_markers: Vec, - model: (String, Bound<'_, PyAny>, Bound<'_, PyAny>), -) -> PyResult<(Option, Option)> { - let (name, materialized, cursor) = model; - let incremental: bool = materialized - .downcast::() - .is_ok_and(|text| text.to_string_lossy() == INCREMENTAL_MATERIALIZATION); - let cursor: Option = match cursor.downcast::() { +/// The reserved executor bound markers no authored SQL may contain. +pub(crate) fn reserved_cursor_markers() -> Vec { + RESERVED_CURSOR_MARKERS + .iter() + .map(|marker| (*marker).to_owned()) + .collect() +} + +/// A `CompileInputError` carrying `message`. +fn compile_input_error(py: Python<'_>, message: String) -> PyResult { + Ok(PyErr::from_value( + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?.call1((message,))?, + )) +} + +/// `text` when it is a string, else `None`. +fn text_value(value: &Bound<'_, PyAny>) -> Option { + match value.downcast::() { Ok(text) => Some(text.to_string_lossy().into_owned()), Err(_) => None, - }; - compiler_guard(|| { - Ok( - match validated_model_intrinsics( - sql, - &reserved_markers, - &IntrinsicModel { - name: &name, - incremental, - cursor: cursor.as_deref(), - }, - ) { - Ok(canonical) => (Some(canonical), None), - Err(message) => (None, Some(message)), - }, - ) - }) + } } -/// `sql` with intrinsic calls replaced and whether any was, or the error for `context`. -#[pyfunction] -fn replace_cursor_intrinsics( +/// `sql` with each intrinsic replaced by its bound and whether any was, raising for `context`. +fn replaced( + py: Python<'_>, sql: &str, context: &str, - replacements: (String, String), -) -> PyResult<(String, bool, Option)> { - let (start, end) = replacements; - compiler_guard(|| { - Ok( - match replace_intrinsics(sql, context, |name| { - if name == CURSOR_START { - start.clone() - } else { - end.clone() - } - }) { - Ok((replaced, found)) => (replaced, found, None), - Err(message) => (String::new(), false, Some(message)), + (start, end): (&str, &str), +) -> PyResult<(String, bool)> { + let outcome: Result<(String, bool), String> = compiler_guard(|| { + Ok(replace_intrinsics(sql, context, |name| { + if name == CURSOR_START { + start.to_owned() + } else { + end.to_owned() + } + })) + })?; + outcome.or_else(|message| Err(compile_input_error(py, message)?)) +} + +/// One model query with canonical intrinsic calls, raising when the model may not use them. +pub(crate) fn validated_model_cursor_sql( + py: Python<'_>, + sql: &str, + (model_name, config_values): (&str, &Bound<'_, PyAny>), +) -> PyResult { + let incremental: bool = text_value(&config_values.call_method1("get", ("materialized",))?) + .is_some_and(|materialized| materialized == INCREMENTAL_MATERIALIZATION); + let cursor: Option = text_value(&config_values.call_method1("get", ("cursor",))?); + let reserved_markers: Vec = reserved_cursor_markers(); + let outcome: Result = compiler_guard(|| { + Ok(validated_model_intrinsics( + sql, + &reserved_markers, + &IntrinsicModel { + name: model_name, + incremental, + cursor: cursor.as_deref(), }, - ) - }) + )) + })?; + outcome.or_else(|message| Err(compile_input_error(py, message)?)) +} + +/// `sql` with intrinsics replaced by stable typed literals, only for static SQL analysis. +pub(crate) fn analysis_cursor_sql( + py: Python<'_>, + sql: &str, + cursor_type: &Bound<'_, PyAny>, +) -> PyResult { + let literal: &str = if text_value(cursor_type).as_deref() == Some(INTEGER_CURSOR_TYPE) { + INTEGER_ANALYSIS_BOUND + } else { + TIMESTAMP_ANALYSIS_BOUND + }; + Ok(replaced(py, sql, MODEL_SQL_CONTEXT, (literal, literal))?.0) +} + +/// One model query with canonical intrinsic calls, raising `CompileInputError` when invalid. +#[pyfunction] +#[pyo3(signature = (*, sql, config_values, model_name))] +fn validated_model_cursor_intrinsics( + py: Python<'_>, + sql: &str, + config_values: &Bound<'_, PyAny>, + model_name: &str, +) -> PyResult { + validated_model_cursor_sql(py, sql, (model_name, config_values)) +} + +/// `sql` with intrinsics replaced by typed literals of `cursor_type` for static analysis. +#[pyfunction] +#[pyo3(signature = (*, sql, cursor_type))] +fn cursor_intrinsics_analysis_sql( + py: Python<'_>, + sql: &str, + cursor_type: &Bound<'_, PyAny>, +) -> PyResult { + analysis_cursor_sql(py, sql, cursor_type) +} + +/// Whether `sql` uses cursor intrinsics, and `sql` with both bounds rendered when given. +#[pyfunction] +#[pyo3(signature = (*, sql, start_sql=None, end_sql=None))] +fn resolve_cursor_intrinsics( + py: Python<'_>, + sql: &str, + start_sql: Option<&str>, + end_sql: Option<&str>, +) -> PyResult<(String, bool)> { + let (rendered, found) = replaced( + py, + sql, + SQL_CONTEXT, + ( + start_sql.unwrap_or(CURSOR_START_CALL), + end_sql.unwrap_or(CURSOR_END_CALL), + ), + )?; + if !found || start_sql.is_none() || end_sql.is_none() { + return Ok((sql.to_owned(), found)); + } + Ok((rendered, true)) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(validated_model_cursor_intrinsics, module)?)?; - module.add_function(wrap_pyfunction!(replace_cursor_intrinsics, module)?)?; + module.add_function(wrap_pyfunction!(cursor_intrinsics_analysis_sql, module)?)?; + module.add_function(wrap_pyfunction!(resolve_cursor_intrinsics, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index e208d7f7bb..e4a43d5838 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -4,6 +4,7 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; +use crate::bindings::_helpers::attachments::cursor_intrinsics::reserved_cursor_markers; use crate::bindings::_helpers::compile_attachment::model_config::native_model_config_session; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, @@ -21,7 +22,6 @@ const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const VALIDATION_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.validation"; const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; -const CURSOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; const TEMPLATING_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.templating"; const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; @@ -291,7 +291,7 @@ pub(crate) fn build_model_inputs<'py>( classes, ), syntax.getattr("native_mapping")?, - imported(py, CURSOR_MODULE, "cursor_intrinsic_semantics")?.call0()?, + reserved_cursor_markers(), ( ( settings.getattr("sql_analysis")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs index 87816897fb..5b67c33912 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs @@ -3,6 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; +use crate::bindings::_helpers::attachments::cursor_intrinsics::validated_model_cursor_sql; use crate::bindings::_helpers::compile_driver::assembly_project::ADAPTER_NAME_MODULE; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; @@ -10,8 +11,6 @@ const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const CURSOR_INTRINSICS_MODULE: &str = - "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; /// The `CompiledSqlScenario` fields copied from its input, in Python's keyword order. const SCENARIO_FIELDS: [&str; 12] = [ "sql_body", @@ -74,11 +73,6 @@ pub(crate) fn test_model_query_overrides<'py>( ], )?; let expand: Bound<'py, PyAny> = py.import(NATIVE_MODULE)?.getattr("expand_sql_macros")?; - let intrinsics: Bound<'py, PyAny> = imported( - py, - CURSOR_INTRINSICS_MODULE, - "get_validated_model_cursor_intrinsics", - )?; for model_input in inputs.getattr("model_inputs")?.try_iter()? { let model_input: Bound<'py, PyAny> = model_input?; let file_path: Bound<'py, PyAny> = @@ -102,16 +96,13 @@ pub(crate) fn test_model_query_overrides<'py>( ], )? .getattr("sql")?; - let query: Bound<'py, PyAny> = call_with( - &intrinsics, - &[ - ("sql", expanded), - ( - "config_values", - model_input.getattr("config")?.getattr("values")?, - ), - ("model_name", model_name.clone()), - ], + let query: String = validated_model_cursor_sql( + py, + &expanded.extract::()?, + ( + &model_name.extract::()?, + &model_input.getattr("config")?.getattr("values")?, + ), )?; overrides.set_item(model_name, query)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index 0e23885df8..321999347e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -3,6 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; +use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::compile_driver::assembly_analysis::{ ANALYSIS_MODELS_MODULE, AnalysisContext, model_stem, selected_for_analysis, @@ -12,15 +13,12 @@ use crate::bindings::_helpers::hand_built::compiled_model::compiled_model as han use crate::bindings::_helpers::render::macro_calls::top_level_macro_call_names; const NATIVE_MODULE: &str = "sqlbuild._native"; -const CURSOR_INTRINSICS_MODULE: &str = - "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; /// The Python stages each model's assembly calls, looked up once per project. struct ModelStages<'py> { - analysis_sql: Bound<'py, PyAny>, placeholders: Bound<'py, PyAny>, validate_hooks: Bound<'py, PyAny>, validate_sql: Bound<'py, PyAny>, @@ -44,11 +42,6 @@ pub(crate) fn compiled_models<'py>( let py: Python<'py> = inputs.py(); let native: Bound<'py, PyAny> = py.import(NATIVE_MODULE)?.into_any(); let stages: ModelStages<'py> = ModelStages { - analysis_sql: imported( - py, - CURSOR_INTRINSICS_MODULE, - "cursor_intrinsics_analysis_sql", - )?, placeholders: native.getattr("model_placeholders")?, validate_hooks: native.getattr("validate_hook_sql_syntax")?, validate_sql: native.getattr("validate_sql_syntax")?, @@ -83,16 +76,15 @@ fn compiled_model<'py>( let model_name: Bound<'py, PyAny> = model_stem(model_input)?; let syntax_validated: bool = request.native_model.getattr("syntax_valid")?.is_truthy()?; let config_values: Bound<'py, PyAny> = model_input.getattr("config")?.getattr("values")?; - let analysis_query_sql: Bound<'py, PyAny> = call_with( - &stages.analysis_sql, - &[ - ("sql", model_input.getattr("query_sql")?), - ( - "cursor_type", - config_values.call_method1("get", ("cursor_type",))?, - ), - ], - )?; + let analysis_query_sql: Bound<'py, PyAny> = PyString::new( + py, + &analysis_cursor_sql( + py, + &model_input.getattr("query_sql")?.extract::()?, + &config_values.call_method1("get", ("cursor_type",))?, + )?, + ) + .into_any(); let placeholders: Bound<'py, PyAny> = stages.placeholders.call1((model_input,))?; let input_validated: bool = model_input.getattr("sql_validation_enabled")?.is_truthy()?; let location: Bound<'py, PyTuple> = PyTuple::new( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs index 80fd22543b..3ba20fef2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs @@ -5,11 +5,9 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::project_assembly::syntax_validation::hook_sql_statements; -const CURSOR_INTRINSICS_MODULE: &str = - "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; fn config_value<'py>(model_input: &Bound<'py, PyAny>, key: &str) -> PyResult> { @@ -46,17 +44,15 @@ pub(crate) fn model_placeholders<'py>( /// The model query with cursor intrinsics replaced by typed literals for static analysis. fn analysis_sql<'py>(model_input: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = model_input.py(); - call_with( - &imported( + Ok(PyString::new( + py, + &analysis_cursor_sql( py, - CURSOR_INTRINSICS_MODULE, - "cursor_intrinsics_analysis_sql", + &model_input.getattr("query_sql")?.extract::()?, + &config_value(model_input, "cursor_type")?, )?, - &[ - ("sql", model_input.getattr("query_sql")?), - ("cursor_type", config_value(model_input, "cursor_type")?), - ], ) + .into_any()) } fn syntax_checks<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs index aef978d028..75420b0b46 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs @@ -9,6 +9,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::analysis::binding_catalog::{ self, CASE_SENSITIVE_BINDING_DIALECTS, NATIVE_DIALECT_ALIASES, schema_validations, }; +use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; use crate::bindings::_helpers::semantic_shapes::shapes::{ @@ -23,8 +24,6 @@ const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const BINDING_UNKNOWN_TABLE_INTERNAL_CODE: &str = "B000"; const BINDING_REQUESTS_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.binding_requests"; -const CURSOR_INTRINSICS_MODULE: &str = - "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; const DATACLASSES_MODULE: &str = "dataclasses"; const GRAPHLIB_MODULE: &str = "graphlib"; const REF_KIND: &str = "ref"; @@ -270,25 +269,21 @@ fn validated_diagnostics<'py>( let request_builder: Object<'py> = py .import(BINDING_REQUESTS_MODULE)? .getattr("get_complete_schema_binding_request")?; - let cursor_sql: Object<'py> = py - .import(CURSOR_INTRINSICS_MODULE)? - .getattr("cursor_intrinsics_analysis_sql")?; let ignores: Object<'py> = ignores_quoted_case(project, dialect)?; let mut requests: Vec> = Vec::with_capacity(models.len()); for model in models { - let query_sql: Object<'py> = call( - &cursor_sql, - &[ - ("sql", model.getattr("query_sql")?), - ( - "cursor_type", - model - .getattr("config")? - .getattr("values")? - .call_method1("get", ("cursor_type",))?, - ), - ], - )?; + let query_sql: Object<'py> = PyString::new( + py, + &analysis_cursor_sql( + py, + &model.getattr("query_sql")?.extract::()?, + &model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("cursor_type",))?, + )?, + ) + .into_any(); let request: Object<'py> = call( &request_builder, &[ diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 04d0a6a207..e0fb9d6af0 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2707,13 +2707,12 @@ def scope_test_ctes( def omitted_ceremonial_select(sql: str, syntax: dict[str, object]) -> int | None: ... def extract_sql_tests_json(request_json: str) -> str: ... def validated_model_cursor_intrinsics( - sql: str, - reserved_markers: list[str], - model: tuple[str, object, object], -) -> tuple[str | None, str | None]: ... -def replace_cursor_intrinsics( - sql: str, context: str, replacements: tuple[str, str] -) -> tuple[str, bool, str | None]: ... + *, sql: str, config_values: Mapping[str, object], model_name: str +) -> str: ... +def cursor_intrinsics_analysis_sql(*, sql: str, cursor_type: object) -> str: ... +def resolve_cursor_intrinsics( + *, sql: str, start_sql: str | None = None, end_sql: str | None = None +) -> tuple[str, bool]: ... def parse_macro_arguments(text: str, nested: list[tuple[int, int]]) -> tuple[object, ...]: ... def scan_sql_declaration_references( sqls: list[str], diff --git a/src/sqlbuild/compiler/compile/_helpers/render/cursor_intrinsics.py b/src/sqlbuild/compiler/compile/_helpers/render/cursor_intrinsics.py deleted file mode 100644 index f734a3b99a..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/render/cursor_intrinsics.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Recognition and rendering for model cursor-bound SQL intrinsics.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.planner.constants import ( - MICROBATCH_END_SENTINEL, - MICROBATCH_START_SENTINEL, -) -from sqlbuild.compiler.planner.types import CursorType - -_RESERVED_MARKERS: list[str] = [MICROBATCH_START_SENTINEL, MICROBATCH_END_SENTINEL] -_CURSOR_START_CALL: str = "__cursor_start()" -_CURSOR_END_CALL: str = "__cursor_end()" - - -def get_validated_model_cursor_intrinsics( - *, sql: str, config_values: dict[str, object], model_name: str -) -> str: - """Validate and canonicalize intrinsics in one model query.""" - - canonical_sql, error = _native.validated_model_cursor_intrinsics( - sql, - _RESERVED_MARKERS, - (model_name, config_values.get("materialized"), config_values.get("cursor")), - ) - if error is not None: - raise CompileInputError(error) - return str(canonical_sql) - - -def render_cursor_intrinsics(*, sql: str, start_sql: str, end_sql: str) -> str: - """Render recognized intrinsics to complete adapter-specific bound expressions.""" - - rendered, _ = _replaced(sql=sql, context="Model SQL", start_sql=start_sql, end_sql=end_sql) - return rendered - - -def cursor_intrinsics_analysis_sql(*, sql: str, cursor_type: object) -> str: - """Replace intrinsics with stable typed literals only for static SQL analysis.""" - - literal: str = ( - "CAST(0 AS BIGINT)" - if cursor_type == CursorType.INTEGER - else "CAST('2000-01-01 00:00:00' AS TIMESTAMP)" - ) - return render_cursor_intrinsics(sql=sql, start_sql=literal, end_sql=literal) - - -def has_cursor_intrinsics(sql: str) -> bool: - """Return whether executable SQL contains either cursor intrinsic.""" - - _, found = _replaced( - sql=sql, context="SQL", start_sql=_CURSOR_START_CALL, end_sql=_CURSOR_END_CALL - ) - return found - - -def _replaced(*, sql: str, context: str, start_sql: str, end_sql: str) -> tuple[str, bool]: - replaced, found, error = _native.replace_cursor_intrinsics(sql, context, (start_sql, end_sql)) - if error is not None: - raise CompileInputError(error) - return replaced, found - - -def cursor_intrinsic_semantics() -> list[str]: - """The reserved markers native cursor-intrinsic validation reads.""" - - return list(_RESERVED_MARKERS) diff --git a/src/sqlbuild/compiler/compile/main/cursor_intrinsics.py b/src/sqlbuild/compiler/compile/main/cursor_intrinsics.py deleted file mode 100644 index 2587bdf5ee..0000000000 --- a/src/sqlbuild/compiler/compile/main/cursor_intrinsics.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Published cursor-intrinsic compiler operation.""" - -from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( - has_cursor_intrinsics, - render_cursor_intrinsics, -) - - -def resolve_cursor_intrinsics( - *, sql: str, start_sql: str | None = None, end_sql: str | None = None -) -> tuple[str, bool]: - """Report intrinsic use and optionally render both bounds.""" - - found: bool = has_cursor_intrinsics(sql) - if not found or start_sql is None or end_sql is None: - return sql, found - return ( - render_cursor_intrinsics(sql=sql, start_sql=start_sql, end_sql=end_sql), - True, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py index 7bbffc5884..b819708b61 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py @@ -3,8 +3,8 @@ from __future__ import annotations from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token +from sqlbuild._native import resolve_cursor_intrinsics from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics from sqlbuild.compiler.planner.models import CursorBounds diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py index eee9f5e4be..2a1d38d872 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py @@ -11,10 +11,10 @@ get_config_str, resolve_bounded_cursor_override, resolve_cursor_input_roles, + resolve_cursor_intrinsics, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationLookup -from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics from sqlbuild.compiler.compile.models import ( CompiledRelationLocation, CursorInputRoles, diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py index 3df6a798ef..31bf70ada0 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py @@ -8,10 +8,9 @@ from sqlbuild._native import cursor_floor_to_grain as floor_to_grain from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render -from sqlbuild._native import get_config_str +from sqlbuild._native import get_config_str, resolve_cursor_intrinsics from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics from sqlbuild.compiler.compile.models import CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner._helpers.resolve.cursor_intrinsics import ( diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index 92972ec941..65f2ab648f 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -4,10 +4,9 @@ from collections.abc import Callable, Mapping -from sqlbuild._native import assert_no_unresolved_sql_markers +from sqlbuild._native import assert_no_unresolved_sql_markers, resolve_cursor_intrinsics from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL from sqlbuild.compiler.planner.main.changes.relation_replacement import ( diff --git a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py index e8503ef03f..e611499f86 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py +++ b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py @@ -22,9 +22,9 @@ from sqlbuild._native import cursor_render as render from sqlbuild._native import cursor_sentinel_from_token as sentinel_from_token from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token +from sqlbuild._native import resolve_cursor_intrinsics from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists -from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL from sqlbuild.compiler.planner.main.cursor_window.cursor_window_override_flags import ( cursor_window_override_flags, diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 3594df7e8a..5341e0102e 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -19,6 +19,7 @@ from sqlbuild._native import ( analyze_native_model_sql, build_compile_inputs, + cursor_intrinsics_analysis_sql, discover_project_inputs, infer_native_expression_source_shapes, model_placeholders, @@ -38,9 +39,6 @@ ) from sqlbuild.compiler.analysis_session.types import ProofRow from sqlbuild.compiler.compile._helpers.analysis.cache import build_analysis_cache_context -from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( - cursor_intrinsics_analysis_sql, -) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( AnalysisCacheContext, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py index 8520ec0c33..6fd7f9f166 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py @@ -2,14 +2,14 @@ import pytest -from sqlbuild._native import infer_native_query_columns -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery -from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( +from sqlbuild._native import ( cursor_intrinsics_analysis_sql, - get_validated_model_cursor_intrinsics, - render_cursor_intrinsics, + infer_native_query_columns, + resolve_cursor_intrinsics, + validated_model_cursor_intrinsics, ) +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.analysis_session.models import NativeColumnQuery from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.lineage.types import InferredNullability @@ -35,7 +35,7 @@ def test_given_valid_incremental_query_when_validating_then_canonicalizes_intrinsics( test_case: CursorIntrinsicValidationTestCase, ) -> None: - sql: str = get_validated_model_cursor_intrinsics( + sql: str = validated_model_cursor_intrinsics( sql=test_case.sql, config_values={"materialized": "incremental", "cursor": "event_time"}, model_name="events", @@ -72,7 +72,7 @@ def test_given_valid_incremental_query_when_validating_then_canonicalizes_intrin def test_given_comments_and_literals_when_rendering_then_only_replaces_executable_calls( test_case: CursorIntrinsicRenderTestCase, ) -> None: - sql: str = render_cursor_intrinsics( + sql, _ = resolve_cursor_intrinsics( sql=test_case.sql, start_sql="TIMESTAMP '2026-01-01'", end_sql="TIMESTAMP '2026-01-02'", @@ -106,7 +106,7 @@ def test_given_invalid_intrinsic_when_validating_then_raises( test_case: CursorIntrinsicErrorTestCase, ) -> None: with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - get_validated_model_cursor_intrinsics( + validated_model_cursor_intrinsics( sql=test_case.sql, config_values={"materialized": "incremental", "cursor": "event_time"}, model_name="events", From d5b877a163a23f35307861fba137435ca8246ae5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 03:29:17 +0100 Subject: [PATCH 07/40] refactor: expand compile templates and resolve run ids natively Template data, effective vars, early model templates, template read recording and run ids now run in the Rust glue; Python callers use the native expand_template_data, expand_effective_vars, record_template_reads and resolve_run_id bindings. --- .../_helpers/command_runtime/warehouses.rs | 2 +- .../compile_attachment/effective_config.rs | 38 ++-- .../compile_attachment/model_inputs.rs | 3 +- .../_helpers/compile_attachment/target.rs | 25 +- .../compile_driver/assembly_project.rs | 28 +-- .../_helpers/compile_driver/inputs.rs | 8 +- .../_helpers/compile_driver/targets.rs | 2 +- .../_helpers/model_config/config_templates.rs | 75 +++--- .../src/bindings/_helpers/model_config/mod.rs | 1 + .../_helpers/model_config/registration.rs | 6 +- .../_helpers/model_config/template_data.rs | 213 ++++++++++++++++++ src/sqlbuild/_native.pyi | 20 +- .../_helpers/render/context_templates.py | 46 ---- .../compile/_helpers/render/sql_vars.py | 3 +- .../compile/_helpers/render/templating.py | 82 ------- .../compile/main/effective_runtime.py | 2 +- .../compile/main/expand_template_data.py | 30 --- .../main/_expand_native_config_templates.py | 36 --- src/sqlbuild/compiler/model_config/models.py | 32 --- .../integrations/dlt/_helpers/config.py | 2 +- .../integrations/ingestr/_helpers/command.py | 2 +- .../main/compile/test_native_config_errors.py | 4 +- .../sqlbuild/compiler/model_config/helpers.py | 2 +- .../model_config/test_native_model_config.py | 2 +- .../compile/_helpers/config/helpers.py | 2 +- .../compile/_helpers/test_templating.py | 2 +- .../compile/test_effective_runtime.py | 2 +- 27 files changed, 305 insertions(+), 365 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs delete mode 100644 src/sqlbuild/compiler/compile/_helpers/render/context_templates.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/render/templating.py delete mode 100644 src/sqlbuild/compiler/compile/main/expand_template_data.py delete mode 100644 src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py delete mode 100644 src/sqlbuild/compiler/model_config/models.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index daff68d0bd..6677cb91ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -30,7 +30,7 @@ const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_na const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main._session_warehouse_support"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; -const EXPAND_TEMPLATE_MODULE: &str = "sqlbuild.compiler.compile.main.expand_template_data"; +const EXPAND_TEMPLATE_MODULE: &str = "sqlbuild._native"; const RESOLVE_OPTIONS: (&[&str], &[&str]) = ( &[ "discovered_inputs", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs index bff1441a51..a213be3c23 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs @@ -2,13 +2,17 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyfunction; -use pyo3::types::{PyBool, PyDict, PyList, PyString}; +use pyo3::types::{PyDict, PyList}; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use sqlbuild_model_config::templates::models::TemplateOptions; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::model_config::template_data::{ + expanded_effective_vars, expanded_template_data, +}; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const TEMPLATING_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.templating"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; const CONNECTION_LABEL: &str = "effective connection"; @@ -31,21 +35,17 @@ pub(crate) fn build_effective_connection<'py>( .call1((error.unwrap_or_default(),))?; return Err(PyErr::from_value(raised)); } - let falsy = || PyBool::new(py, false).to_owned().into_any(); - call_with( - &imported(py, TEMPLATING_MODULE, "expand_template_data")?, - &[ - ("value", connection), - ("variables", effective_vars.clone()), - ("context_values", PyDict::new(py).into_any()), - ( - "context_label", - PyString::new(py, CONNECTION_LABEL).into_any(), - ), - ("allow_context", falsy()), - ("preserve_context_tokens", falsy()), - ("preserve_unknown_context", falsy()), - ], + expanded_template_data( + &connection, + (effective_vars.downcast()?, &PyDict::new(py)), + ( + TemplateOptions { + allow_context: false, + preserve_context_tokens: false, + preserve_unknown_context: false, + }, + CONNECTION_LABEL, + ), ) } @@ -88,7 +88,7 @@ pub(crate) fn build_effective_vars<'py>( let py: Python<'py> = project_config.py(); let merged: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "effective_config_vars")? .call1(((project_config, local_config), target_config, cli_vars))?; - imported(py, TEMPLATING_MODULE, "expand_effective_vars")?.call1((merged,)) + expanded_effective_vars(merged.downcast()?) } /// `build_effective_vars` for a discovered project and target, without CLI vars. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index e4a43d5838..96ace4309a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -23,7 +23,6 @@ const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const VALIDATION_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.validation"; const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; -const TEMPLATING_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.templating"; const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; const PROJECT_VARS_MODULE: &str = "sqlbuild._native"; const MACRO_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; @@ -252,7 +251,7 @@ pub(crate) fn build_model_inputs<'py>( ), ( "record_template_reads", - imported(py, TEMPLATING_MODULE, "record_template_reads")?, + imported(py, NATIVE_MODULE, "record_template_reads")?, ), ( "visible_declarations", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs index cabc68551f..1904f73910 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -8,14 +8,13 @@ use crate::bindings::_helpers::compile_attachment::effective_config::project_var use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::model_config::template_data::{early_model_templates, run_id}; const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const FRONTIER_CONSTANTS_MODULE: &str = "sqlbuild.compiler.frontier.constants"; const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; -const CONTEXT_TEMPLATES_MODULE: &str = - "sqlbuild.compiler.compile._helpers.render.context_templates"; const DATABASE_KEY: &str = "database"; const SCHEMA_KEY: &str = "schema"; @@ -217,24 +216,10 @@ pub(crate) fn build_effective_target_namespace<'py>( SCHEMA_KEY, first_truthy([target_value(SCHEMA_KEY)?, connection_schema, default_schema])?, )?; - let run_id: Bound<'py, PyAny> = call_with( - &imported(py, CONTEXT_TEMPLATES_MODULE, "resolve_run_id")?, - &[("selected_run_id", py.None().into_bound(py))], - )?; - let resolved: Bound<'py, PyDict> = call_with( - &imported( - py, - CONTEXT_TEMPLATES_MODULE, - "resolve_early_model_templates", - )?, - &[ - ("values", values.into_any()), - ("effective_vars", effective_vars), - ("effective_target_name", target_name.clone()), - ("run_id", run_id), - ], - )? - .downcast_into()?; + let run_id: Bound<'py, PyAny> = run_id(&py.None().into_bound(py))?; + let resolved: Bound<'py, PyDict> = + early_model_templates(values.as_any(), &effective_vars, (&target_name, &run_id))? + .downcast_into()?; PyTuple::new( py, [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index 5f71574f50..c4cf4b2e98 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -16,12 +16,11 @@ use crate::bindings::_helpers::compile_driver::assembly_models::{ binding_rows, compiled_models, project_binding_diagnostics, }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::model_config::template_data::early_model_templates; use crate::bindings::_helpers::scopes::scope_errors::report_scope_index_errors; const NATIVE_MODULE: &str = "sqlbuild._native"; const SCOPE_USAGES_MODULE: &str = "sqlbuild.compiler.compile.main._scope_index_with_compile_usages"; -const CONTEXT_TEMPLATES_MODULE: &str = - "sqlbuild.compiler.compile._helpers.render.context_templates"; pub(crate) const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; const ADAPTER_TYPES_MODULE: &str = "sqlbuild.adapter.contract.types"; @@ -176,24 +175,13 @@ fn effective_target_values<'py>(inputs: &Bound<'py, PyAny>) -> PyResult( let discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; let adapter_context: Bound<'py, PyAny> = arguments.value("adapter_context")?; let target: TargetContext<'py> = target_context(arguments, &discovered, &adapter_context)?; - let run_id: Bound<'py, PyAny> = call_with( - &imported(py, CONTEXT_TEMPLATES_MODULE, "resolve_run_id")?, - &[("selected_run_id", arguments.value("run_id")?)], - )?; + let run_id: Bound<'py, PyAny> = run_id(&arguments.value("run_id")?)?; let loaded_macros: Bound<'py, PyAny> = imported(py, MACROS_MODULE, "load_project_macros")? .call1((discovered.getattr("macro_files")?,))?; let syntax: Bound<'py, PyAny> = adapter_context.getattr("sql_lexical_syntax")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index 919ddc74f3..1d58698350 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -27,7 +27,7 @@ const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_na const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; -const EXPAND_TEMPLATE_MODULE: &str = "sqlbuild.compiler.compile.main.expand_template_data"; +const EXPAND_TEMPLATE_MODULE: &str = "sqlbuild._native"; const SETTING_HELP_MODULE: &str = "sqlbuild.errors.setting_help.main.setting_help"; /// Connection settings that identify the warehouse two targets write to. const CONNECTION_IDENTITY_KEYS: [&str; 7] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs index 4efb336efe..e0375e37ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs @@ -3,12 +3,12 @@ use std::cell::RefCell; use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyResult, Python}; use pyo3::types::{ PyBool, PyBoolMethods, PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, PyTupleMethods, }; -use pyo3::{FromPyObject, IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; +use pyo3::{FromPyObject, PyErr}; use sqlbuild_model_config::errors::ConfigError; use sqlbuild_model_config::templates::errors::TemplateError; use sqlbuild_model_config::templates::main::expand_template_string::expand_template_string; @@ -20,6 +20,7 @@ use sqlbuild_model_config::templates::models::{ use sqlbuild_model_config::templates::types::TemplateHost; use crate::bindings::_helpers::model_config::config_errors::native_config_error; +use crate::bindings::_helpers::model_config::errors::NativeConfigError; const TEMPLATE_OPEN_TOKEN: &str = "${"; const ENVIRONMENT_READ: &str = "env"; @@ -155,10 +156,6 @@ pub(crate) struct TemplateSources<'py>( pub(crate) Bound<'py, PyDict>, ); -/// `allow_context`, `preserve_context_tokens`, `preserve_unknown_context` and the context label. -#[derive(FromPyObject)] -struct TemplateFlags(bool, bool, bool, String); - /// A template error, or a Python error raised while reading values or building containers. pub(crate) enum Stop { Failure(TemplateError), @@ -268,31 +265,28 @@ impl<'py> TemplateHost for PythonHost<'py> { } } -/// Return `(value, reads)` as `expand_template_data` would, or `(error, reads)`. -#[pyfunction] -fn expand_config_templates<'py>( +/// The expanded value, or the native error expansion stopped at, and the reads in order. +pub(crate) type TemplateOutcome<'py> = ( + Result, Py>, + Vec<(&'static str, String)>, +); + +/// `value` with every template expanded under `options`, or the error, with its reads. +pub(crate) fn expand_config_templates<'py>( py: Python<'py>, - value: Bound<'py, PyAny>, + value: &Bound<'py, PyAny>, sources: TemplateSources<'py>, - flags: TemplateFlags, -) -> PyResult> { - let TemplateFlags(allow_context, preserve_context_tokens, preserve_unknown_context, label) = - flags; - let options = TemplateOptions { - allow_context, - preserve_context_tokens, - preserve_unknown_context, - }; - let host = PythonHost::new(py, sources, &label); - let result = expanded(&host, &value, options); - outcome(py, host, result, &label) + (options, label): (TemplateOptions, &str), +) -> PyResult> { + let host = PythonHost::new(py, sources, label); + let result = expanded(&host, value, options); + outcome(py, host, result, label) } /// The first `(var name, value path, UTF-8 byte offset)` of a lone surrogate in raw vars. -#[pyfunction] -fn lone_surrogate_site( - raw_values: Bound<'_, PyDict>, -) -> PyResult, String, usize)>> { +pub(crate) fn lone_surrogate_site<'py>( + raw_values: &Bound<'py, PyDict>, +) -> PyResult, String, usize)>> { for (key, value) in raw_values.iter() { if let Some((path, offset)) = surrogate_path(&value, VALUE_PATH.to_owned())? { return Ok(Some((key.str()?, path, offset))); @@ -348,13 +342,12 @@ fn first_surrogate_offset(text: &Bound<'_, PyString>) -> PyResult> Ok(None) } -/// Return `(resolved vars, reads)` as `expand_effective_vars` would, or `(error, reads)`. -#[pyfunction] -fn expand_effective_vars<'py>( +/// The resolved effective vars, or the native error resolution stopped at, with the reads. +pub(crate) fn resolve_effective_vars<'py>( py: Python<'py>, - raw_values: Bound<'py, PyDict>, + raw_values: &Bound<'py, PyDict>, environment: Bound<'py, PyAny>, -) -> PyResult> { +) -> PyResult> { let mut host = PythonHost::new( py, TemplateSources(raw_values.clone(), environment, PyDict::new(py)), @@ -381,18 +374,13 @@ fn outcome<'py>( host: PythonHost<'py>, result: Result, Stop>, label: &str, -) -> PyResult> { +) -> PyResult> { let value = match result { - Ok(value) => value, - Err(Stop::Failure(error)) => native_config_error(py, template_error(&error, label))? - .into_pyobject(py)? - .into_any(), + Ok(value) => Ok(value), + Err(Stop::Failure(error)) => Err(native_config_error(py, template_error(&error, label))?), Err(Stop::Python(error)) => return Err(error), }; - Ok((value, host.into_reads()) - .into_pyobject(py)? - .into_any() - .unbind()) + Ok((value, host.into_reads())) } /// Expand every template in `value` and the containers inside it. @@ -443,10 +431,3 @@ pub(crate) fn expanded<'py>( } Ok(value.clone()) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(expand_config_templates, module)?)?; - module.add_function(wrap_pyfunction!(expand_effective_vars, module)?)?; - module.add_function(wrap_pyfunction!(lone_surrogate_site, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index bec5690c43..d76eb0322f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -11,3 +11,4 @@ pub(crate) mod model_config_builder; pub(crate) mod model_validator; pub(crate) mod project_defaults; pub(crate) mod registration; +pub(crate) mod template_data; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs index d56dbf58c4..8f9293b80f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs @@ -3,18 +3,18 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::model_config::{ - column_migrations, config_errors, config_templates, config_values, described_models, - header_columns, model_config_builder, model_validator, + column_migrations, config_errors, config_values, described_models, header_columns, + model_config_builder, model_validator, template_data, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { column_migrations::register(module)?; config_errors::register(module)?; - config_templates::register(module)?; config_values::register(module)?; described_models::register(module)?; header_columns::register(module)?; model_config_builder::register(module)?; model_validator::register(module)?; + template_data::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs new file mode 100644 index 0000000000..28dedd70e0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs @@ -0,0 +1,213 @@ +//! Template expansion for compile inputs and effective vars, with the reads it makes recorded. + +use std::time::{SystemTime, UNIX_EPOCH}; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::scenario_command::main::format_captured_at::format_captured_at; +use sqlbuild_model_config::templates::models::TemplateOptions; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::model_config::config_templates::{ + TemplateOutcome, TemplateSources, expand_config_templates, lone_surrogate_site, + resolve_effective_vars, +}; + +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; +const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; +const CONFIG_ERROR_MODULE: &str = "sqlbuild.compiler.model_config.main._native_config_error"; +const UUID_MODULE: &str = "uuid"; +const ENVIRONMENT_READ: &str = "env"; +const RUN_ID_KEY: &str = "run.id"; +const RUN_TARGET_KEY: &str = "run.target"; +const MODEL_CONFIG_LABEL: &str = "model config"; +const RUN_ID_SUFFIX_LENGTH: usize = 12; +const TIMESTAMP_SEPARATORS: [char; 2] = ['-', ':']; + +/// Record native template environment and context reads for project reuse, in order. +pub(crate) fn record_reads<'a>( + py: Python<'_>, + reads: impl IntoIterator, +) -> PyResult<()> { + let registry: Bound<'_, PyAny> = imported(py, COMPILE_CONSTANTS_MODULE, "COMPILE_INPUT_READS")?; + for (kind, name) in reads { + let method: &str = if kind == ENVIRONMENT_READ { + "environment_read" + } else { + "context_read" + }; + registry.call_method1(method, (name,))?; + } + Ok(()) +} + +/// Record `outcome`'s reads, then return its value or raise its error as Python's stage does. +fn applied<'py>(py: Python<'py>, outcome: TemplateOutcome<'py>) -> PyResult> { + let (value, reads) = outcome; + record_reads(py, reads.iter().map(|(kind, name)| (*kind, name.as_str())))?; + match value { + Ok(value) => Ok(value), + Err(error) => Err(PyErr::from_value(call_with( + &imported(py, CONFIG_ERROR_MODULE, "native_config_error")?, + &[("error", error.into_bound(py).into_any())], + )?)), + } +} + +/// The process environment as compile text reads it. +fn environment(py: Python<'_>) -> PyResult> { + imported(py, UNICODE_ENVIRONMENT_MODULE, "UnicodeEnvironment")?.call0() +} + +/// `value` with every `${...}` template expanded, recording the reads it made. +pub(crate) fn expanded_template_data<'py>( + value: &Bound<'py, PyAny>, + (variables, context_values): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), + (options, label): (TemplateOptions, &str), +) -> PyResult> { + let py: Python<'py> = value.py(); + let sources = TemplateSources(variables.clone(), environment(py)?, context_values.clone()); + applied( + py, + expand_config_templates(py, value, sources, (options, label))?, + ) +} + +/// Merged effective vars with recursive `${name}` expansion. +pub(crate) fn expanded_effective_vars<'py>( + raw_values: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = raw_values.py(); + if let Some((name, path, byte_offset)) = lone_surrogate_site(raw_values)? { + let error: Bound<'py, PyAny> = imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")? + .call( + (format!( + "Variable '{name}' holds a lone surrogate in {path} at UTF-8 byte \ + {byte_offset}, which is not valid Unicode text" + ),), + Some(&keywords( + py, + &[( + "help", + PyString::new( + py, + &format!( + "Set '{name}' to valid Unicode text in --vars or the [vars] \ + table; the value is not shown because it may be a secret" + ), + ) + .into_any(), + )], + )?), + )?; + return Err(PyErr::from_value(error)); + } + applied( + py, + resolve_effective_vars(py, raw_values, environment(py)?)?, + ) +} + +/// Model header `values` with `${name}`, `${ENV:...}` and early `run.*` templates resolved. +pub(crate) fn early_model_templates<'py>( + values: &Bound<'py, PyAny>, + effective_vars: &Bound<'py, PyAny>, + (target_name, run_id): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = values.py(); + let context_values: Bound<'py, PyDict> = PyDict::new(py); + context_values.set_item(RUN_ID_KEY, run_id)?; + context_values.set_item(RUN_TARGET_KEY, target_name)?; + expanded_template_data( + values, + (effective_vars.downcast()?, &context_values), + ( + TemplateOptions { + allow_context: true, + preserve_context_tokens: false, + preserve_unknown_context: true, + }, + MODEL_CONFIG_LABEL, + ), + ) +} + +/// The selected run id, or a new `YYYYMMDDTHHMMSSZ_<12 hex>` compile invocation id. +pub(crate) fn run_id<'py>(selected_run_id: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = selected_run_id.py(); + if !selected_run_id.is_none() { + return Ok(selected_run_id.clone()); + } + let epoch_seconds: i64 = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_secs()).unwrap_or(i64::MAX) + }); + let timestamp: String = format_captured_at(epoch_seconds).replace(TIMESTAMP_SEPARATORS, ""); + let suffix: String = imported(py, UUID_MODULE, "uuid4")? + .call0()? + .getattr("hex")? + .extract()?; + let suffix: &str = suffix.get(..RUN_ID_SUFFIX_LENGTH).unwrap_or(&suffix); + Ok(PyString::new(py, &format!("{timestamp}_{suffix}")).into_any()) +} + +/// Recursively expand template strings inside supported Python container values. +#[pyfunction] +#[pyo3(signature = (**request))] +fn expand_template_data<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request: Request<'_, 'py> = Request::new("expand_template_data", request)?; + let flag = |name: &str| -> PyResult { request.get(name)?.is_truthy() }; + expanded_template_data( + &request.get("value")?, + ( + request.get("variables")?.downcast()?, + request.get("context_values")?.downcast()?, + ), + ( + TemplateOptions { + allow_context: flag("allow_context")?, + preserve_context_tokens: flag("preserve_context_tokens")?, + preserve_unknown_context: flag("preserve_unknown_context")?, + }, + &request.get("context_label")?.extract::()?, + ), + ) +} + +/// Resolve merged effective vars with recursive `${name}` expansion. +#[pyfunction] +fn expand_effective_vars<'py>(raw_values: &Bound<'py, PyDict>) -> PyResult> { + expanded_effective_vars(raw_values) +} + +/// Record native template environment and context reads for project reuse, in order. +#[pyfunction] +fn record_template_reads(py: Python<'_>, reads: Vec<(String, String)>) -> PyResult<()> { + record_reads( + py, + reads + .iter() + .map(|(kind, name)| (kind.as_str(), name.as_str())), + ) +} + +/// Resolve a stable compile invocation id. +#[pyfunction] +#[pyo3(signature = (*, selected_run_id))] +fn resolve_run_id<'py>(selected_run_id: &Bound<'py, PyAny>) -> PyResult> { + run_id(selected_run_id) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(expand_template_data, module)?)?; + module.add_function(wrap_pyfunction!(expand_effective_vars, module)?)?; + module.add_function(wrap_pyfunction!(record_template_reads, module)?)?; + module.add_function(wrap_pyfunction!(resolve_run_id, module)?)?; + Ok(()) +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index e0fb9d6af0..fa829070c1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2573,7 +2573,6 @@ def select_latest_fingerprints( node_names: tuple[str, ...], filtered_node_types: tuple[str, ...] = (), ) -> FingerprintSet: ... -def lone_surrogate_site(raw_values: dict[str, object], /) -> tuple[str, str, int] | None: ... def environment_template_names(values: list[str | None], /) -> list[str]: ... def find_python_string_locations( *, @@ -2624,14 +2623,19 @@ def expand_declaration_references( consumer: tuple[object | None, bool, str], renderers: tuple[object, dict[str, object]], ) -> tuple[str, list[tuple[int, int, int, int]], list[Any]]: ... -def expand_config_templates( +def expand_template_data( + *, value: object, - sources: tuple[dict[str, object], object, dict[str, str | None]], - flags: tuple[bool, bool, bool, str], -) -> tuple[object, list[tuple[str, str]]]: ... -def expand_effective_vars( - raw_values: dict[str, object], environment: object -) -> tuple[object, list[tuple[str, str]]]: ... + variables: dict[str, object], + context_values: dict[str, str | None], + context_label: str, + allow_context: bool, + preserve_context_tokens: bool, + preserve_unknown_context: bool, +) -> object: ... +def expand_effective_vars(raw_values: dict[str, object]) -> dict[str, object]: ... +def record_template_reads(reads: Sequence[tuple[str, str]]) -> None: ... +def resolve_run_id(*, selected_run_id: str | None) -> str: ... class NativeModelConfigBuilder: def __init__( diff --git a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py deleted file mode 100644 index 4f6baacade..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Compile-time context and template resolution for model header values.""" - -from __future__ import annotations - -from datetime import UTC, datetime -from typing import cast -from uuid import uuid4 - -from sqlbuild.compiler.compile._helpers.render.templating import expand_template_data -from sqlbuild.compiler.compile.types import CompileContextKey - - -def resolve_early_model_templates( - *, - values: dict[str, object], - effective_vars: dict[str, object], - effective_target_name: str | None, - run_id: str, -) -> dict[str, object]: - """Resolve `${name}`, `${ENV:...}`, and early `run.*` model templates.""" - - return cast( - dict[str, object], - expand_template_data( - value=values, - variables=effective_vars, - context_values={ - CompileContextKey.RUN_ID: run_id, - CompileContextKey.RUN_TARGET: effective_target_name, - }, - context_label="model config", - allow_context=True, - preserve_context_tokens=False, - preserve_unknown_context=True, - ), - ) - - -def resolve_run_id(*, selected_run_id: str | None) -> str: - """Resolve a stable compile invocation id.""" - - if selected_run_id is not None: - return selected_run_id - timestamp_prefix: str = datetime.now(tz=UTC).strftime("%Y%m%dT%H%M%SZ") - unique_suffix: str = uuid4().hex[:12] - return f"{timestamp_prefix}_{unique_suffix}" diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py index 2e3cd8e2e6..a04a888ad3 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py @@ -6,11 +6,10 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import expand_sql_macros, render_project_var_text +from sqlbuild._native import expand_sql_macros, record_template_reads, render_project_var_text from sqlbuild.compiler.compile._helpers.render.declarations import ( expand_scanned_declaration_references, ) -from sqlbuild.compiler.compile._helpers.render.templating import record_template_reads from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( diff --git a/src/sqlbuild/compiler/compile/_helpers/render/templating.py b/src/sqlbuild/compiler/compile/_helpers/render/templating.py deleted file mode 100644 index ccdfcd649b..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/render/templating.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Template expansion for compile inputs, natively, with the reads it makes recorded.""" - -from __future__ import annotations - -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.model_config.constants import ENVIRONMENT_READ -from sqlbuild.compiler.model_config.main._expand_native_config_templates import ( - expand_native_config_templates, -) -from sqlbuild.compiler.model_config.main._native_config_error import native_config_error -from sqlbuild.compiler.model_config.models import ( - NativeTemplateExpansion, - NativeTemplateRejection, - TemplateResolutionFlags, -) - - -def expand_effective_vars(raw_values: dict[str, object]) -> dict[str, object]: - """Resolve merged effective vars with recursive `${name}` expansion.""" - - site: tuple[str, str, int] | None = _native.lone_surrogate_site(raw_values) - if site is not None: - name, path, byte_offset = site - raise CompileInputError( - f"Variable '{name}' holds a lone surrogate in {path} at UTF-8 byte " - f"{byte_offset}, which is not valid Unicode text", - help=( - f"Set '{name}' to valid Unicode text in --vars or the [vars] table; the " - "value is not shown because it may be a secret" - ), - ) - outcome: tuple[object, list[tuple[str, str]]] = _native.expand_effective_vars( - raw_values, UnicodeEnvironment() - ) - record_template_reads(tuple(outcome[1])) - if isinstance(outcome[0], _native.NativeConfigError): - raise native_config_error(error=outcome[0]) - return cast(dict[str, object], outcome[0]) - - -def expand_template_data( - *, - value: object, - variables: dict[str, object], - context_values: dict[str, str | None], - context_label: str, - allow_context: bool, - preserve_context_tokens: bool, - preserve_unknown_context: bool, -) -> object: - """Recursively expand template strings inside supported Python container values.""" - - outcome: NativeTemplateExpansion | NativeTemplateRejection = expand_native_config_templates( - value=value, - variables=variables, - context_values=context_values, - flags=TemplateResolutionFlags( - allow_context=allow_context, - preserve_context_tokens=preserve_context_tokens, - preserve_unknown_context=preserve_unknown_context, - ), - context_label=context_label, - ) - record_template_reads(outcome.reads) - if isinstance(outcome, NativeTemplateRejection): - raise native_config_error(error=outcome.error) - return outcome.value - - -def record_template_reads(reads: tuple[tuple[str, str], ...]) -> None: - """Record native template environment and context reads for project reuse, in order.""" - - for kind, name in reads: - if kind == ENVIRONMENT_READ: - COMPILE_INPUT_READS.environment_read(name) - else: - COMPILE_INPUT_READS.context_read(name) diff --git a/src/sqlbuild/compiler/compile/main/effective_runtime.py b/src/sqlbuild/compiler/compile/main/effective_runtime.py index 719c6713cc..7057413409 100644 --- a/src/sqlbuild/compiler/compile/main/effective_runtime.py +++ b/src/sqlbuild/compiler/compile/main/effective_runtime.py @@ -4,8 +4,8 @@ from sqlbuild._native import ( build_effective_vars, + resolve_run_id, ) -from sqlbuild.compiler.compile._helpers.render.context_templates import resolve_run_id from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name diff --git a/src/sqlbuild/compiler/compile/main/expand_template_data.py b/src/sqlbuild/compiler/compile/main/expand_template_data.py deleted file mode 100644 index cd173d3d2f..0000000000 --- a/src/sqlbuild/compiler/compile/main/expand_template_data.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Public entrypoint for recursive template expansion.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile._helpers.render.templating import ( - expand_template_data as _expand_template_data, -) - - -def expand_template_data( - *, - value: object, - variables: dict[str, object], - context_values: dict[str, str | None], - context_label: str, - allow_context: bool, - preserve_context_tokens: bool, - preserve_unknown_context: bool, -) -> object: - """Recursively expand template strings inside supported Python container values.""" - - return _expand_template_data( - value=value, - variables=variables, - context_values=context_values, - context_label=context_label, - allow_context=allow_context, - preserve_context_tokens=preserve_context_tokens, - preserve_unknown_context=preserve_unknown_context, - ) diff --git a/src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py b/src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py deleted file mode 100644 index c955e8dd0a..0000000000 --- a/src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Expand `${...}` templates natively, returning the value or Python's error with its reads.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment -from sqlbuild.compiler.model_config.models import ( - NativeTemplateExpansion, - NativeTemplateRejection, - TemplateResolutionFlags, -) - - -def expand_native_config_templates( - *, - value: object, - variables: dict[str, object], - context_values: dict[str, str | None], - flags: TemplateResolutionFlags, - context_label: str, -) -> NativeTemplateExpansion | NativeTemplateRejection: - """Return the expanded value, or the error expansion stopped at; both carry their reads.""" - - outcome: tuple[object, list[tuple[str, str]]] = _native.expand_config_templates( - value, - (variables, UnicodeEnvironment(), context_values), - ( - flags.allow_context, - flags.preserve_context_tokens, - flags.preserve_unknown_context, - context_label, - ), - ) - if isinstance(outcome[0], _native.NativeConfigError): - return NativeTemplateRejection(error=outcome[0], reads=tuple(outcome[1])) - return NativeTemplateExpansion(value=outcome[0], reads=tuple(outcome[1])) diff --git a/src/sqlbuild/compiler/model_config/models.py b/src/sqlbuild/compiler/model_config/models.py deleted file mode 100644 index 1a95bb0fbd..0000000000 --- a/src/sqlbuild/compiler/model_config/models.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Natively parsed MODEL header metadata and template expansions.""" - -from __future__ import annotations - -from dataclasses import dataclass - -import sqlbuild._native as _native - - -@dataclass(frozen=True, slots=True) -class NativeTemplateExpansion: - """A natively expanded config value and the environment and context names it read, in order.""" - - value: object - reads: tuple[tuple[str, str], ...] - - -@dataclass(frozen=True, slots=True) -class NativeTemplateRejection: - """Python's exact template error for a config value and the names read before it, in order.""" - - error: _native.NativeConfigError - reads: tuple[tuple[str, str], ...] - - -@dataclass(frozen=True, slots=True) -class TemplateResolutionFlags: - """How one template expansion treats `CTX:` references.""" - - allow_context: bool - preserve_context_tokens: bool - preserve_unknown_context: bool diff --git a/src/sqlbuild/integrations/dlt/_helpers/config.py b/src/sqlbuild/integrations/dlt/_helpers/config.py index 0b316bc06a..9027e67e0d 100644 --- a/src/sqlbuild/integrations/dlt/_helpers/config.py +++ b/src/sqlbuild/integrations/dlt/_helpers/config.py @@ -5,7 +5,7 @@ from dataclasses import replace from typing import cast -from sqlbuild.compiler.compile.main.expand_template_data import expand_template_data +from sqlbuild._native import expand_template_data from sqlbuild.integrations.dlt.models import DltResourceConfig, DltSourceConfig diff --git a/src/sqlbuild/integrations/ingestr/_helpers/command.py b/src/sqlbuild/integrations/ingestr/_helpers/command.py index 6e181b21c9..e9d0d733a7 100644 --- a/src/sqlbuild/integrations/ingestr/_helpers/command.py +++ b/src/sqlbuild/integrations/ingestr/_helpers/command.py @@ -6,7 +6,7 @@ from dataclasses import replace from typing import cast -from sqlbuild.compiler.compile.main.expand_template_data import expand_template_data +from sqlbuild._native import expand_template_data from sqlbuild.integrations.ingestr._helpers.destination import build_destination_uri from sqlbuild.integrations.ingestr.constants import INGESTR_INTEGRATION_KIND from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_config_errors.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_config_errors.py index 90ddd1f640..c893b37765 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_config_errors.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_config_errors.py @@ -7,7 +7,7 @@ import pytest -import sqlbuild.compiler.compile._helpers.render.context_templates as context_templates +import sqlbuild._native as _native from scripts.compiler_differential.constants import FAILURE_BASE_MART from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( MacroExpandedValidatorErrorTestCase, @@ -43,7 +43,7 @@ " materialized incremental,\n incremental_strategy append,\n cursor order_id,\n" " cursor_type integer,\n" ) -_FALLBACKS: tuple[tuple[ModuleType, str], ...] = ((context_templates, "expand_template_data"),) +_FALLBACKS: tuple[tuple[ModuleType, str], ...] = ((_native, "expand_template_data"),) _PROJECT_FILES: dict[str, str] = { "macros/counted.py": _COUNTED_MACRO, "models/marts/customer_totals.sql": _COUNTED_MART, diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index aafc9018a9..20fed3b8c5 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -107,7 +107,7 @@ ) -NATIVE_MODEL_CONFIG_ENTRIES: tuple[str, ...] = ("expand_config_templates",) +NATIVE_MODEL_CONFIG_ENTRIES: tuple[str, ...] = ("expand_template_data",) _NATIVE_ENTRIES: dict[str, Callable[..., object]] = { name: getattr(_native, name) for name in NATIVE_MODEL_CONFIG_ENTRIES } diff --git a/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py index 70bd076c17..d7750bb698 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py @@ -97,7 +97,7 @@ def test_given_template_or_macro_column_metadata_when_attaching_then_loop_expand ModelConfigTierTestCase( description="native model config", expected_native_calls={ - "expand_config_templates": 0, + "expand_template_data": 0, }, ), ], diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py index 0fbec2669a..15e844b23e 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py @@ -8,8 +8,8 @@ build_compile_inputs, discover_project_inputs, native_model_config_session, + record_template_reads, ) -from sqlbuild.compiler.compile._helpers.render.templating import record_template_reads from sqlbuild.compiler.compile.models import ( CompileModelConfig, NativeModelConfigInputs, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py index 90e9062385..b418fe8604 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.compile._helpers.render.templating import ( +from sqlbuild._native import ( expand_effective_vars, expand_template_data, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py b/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py index e14cc9e776..8f38b953d8 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.compile._helpers.render.context_templates import resolve_run_id +from sqlbuild._native import resolve_run_id from sqlbuild.compiler.compile.main.effective_runtime import build_effective_runtime_config from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.models import ( From 74b24be85149927b9d0c31e892c9f48ad1579054 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 03:37:14 +0100 Subject: [PATCH 08/40] refactor: resolve declarations and expand authored SQL natively Declaration scope resolvers, per-path declaration contexts, constant and enum member rendering, project var substitution and authored SQL expansion with spans now run in the Rust glue. The Python wrappers and the unused raw interpolation, declaration scan and declaration expansion bindings go. --- .../compile_attachment/declaration_scope.rs | 13 +- .../compile_attachment/model_inputs.rs | 24 +- .../compile_driver/assembly_checks.rs | 14 +- .../bindings/_helpers/render/authored_sql.rs | 306 +++++++++++++++++ .../_helpers/render/declaration_expansion.rs | 36 +- .../_helpers/render/declaration_values.rs | 315 ++++++++++++++++++ .../_helpers/render/macro_expansion_result.rs | 12 +- .../src/bindings/_helpers/render/mod.rs | 2 + .../bindings/_helpers/render/registration.rs | 7 +- .../sqltext/declaration_references.rs | 30 -- .../bindings/_helpers/sqltext/registration.rs | 4 +- .../bindings/_helpers/sqltext/static_sql.rs | 18 +- src/sqlbuild/_native.pyi | 63 ++-- .../compile/_helpers/render/declarations.py | 222 ------------ .../compile/_helpers/render/sql_vars.py | 138 -------- .../compile/main/expand_sql_with_spans.py | 49 --- .../compile/main/sql_expansion_context.py | 12 +- src/sqlbuild/compiler/compile/models.py | 8 +- .../_scan_native_declaration_references.py | 24 -- src/sqlbuild/compiler/model_loop/types.py | 4 - src/sqlbuild/lint/_helpers/expansion.py | 7 +- .../src/sqlbuild/compiler/scopes/helpers.py | 2 +- .../compiler/compile/_helpers/_test_types.py | 10 - .../compile/_helpers/test_expansion_spans.py | 2 +- .../compile/_helpers/test_sql_vars.py | 62 +--- .../src/sqlbuild/lint/test_expanded_lint.py | 3 +- 26 files changed, 719 insertions(+), 668 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs delete mode 100644 src/sqlbuild/compiler/compile/_helpers/render/declarations.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py delete mode 100644 src/sqlbuild/compiler/compile/main/expand_sql_with_spans.py delete mode 100644 src/sqlbuild/compiler/model_loop/main/_scan_native_declaration_references.py delete mode 100644 src/sqlbuild/compiler/model_loop/types.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs index cb2e2c978e..cce2deebf4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -5,13 +5,13 @@ use pyo3::pyfunction; use pyo3::types::PyTuple; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::render::declaration_values::declaration_scope_resolver; const NATIVE_MODULE: &str = "sqlbuild._native"; const SCAN_CACHE_MODULE: &str = "sqlbuild.compiler.compile.classes.sql_test_scan_cache"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; const OPEN_INDEX_MODULE: &str = "sqlbuild.compiler.scopes.main._open_native_scope_index"; const VALIDATE_INDEX_MODULE: &str = "sqlbuild.compiler.scopes.main._validate_scope_index"; @@ -29,15 +29,8 @@ fn scope_build<'py>( (index, lookup, scan_cache): (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>), ) -> PyResult> { let py: Python<'py> = discovered.py(); - let resolver: Bound<'py, PyAny> = call_with( - &imported(py, DECLARATIONS_MODULE, "build_declaration_scope_resolver")?, - &[ - ("discovered_inputs", discovered.clone()), - ("scope_index", index.clone()), - ("loaded_macros", loaded_macros.clone()), - ("lookup", lookup), - ], - )?; + let resolver: Bound<'py, PyAny> = + declaration_scope_resolver(discovered, &index, (loaded_macros, &lookup))?; call_with( &imported(py, COMPILE_MODELS_MODULE, "DeclarationScopeBuild")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index 96ace4309a..dd1c8999c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -2,7 +2,7 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; -use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; +use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple}; use crate::bindings::_helpers::attachments::cursor_intrinsics::reserved_cursor_markers; use crate::bindings::_helpers::compile_attachment::model_config::native_model_config_session; @@ -10,6 +10,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::compiled_project::loop_classes::model_loop_classes; +use crate::bindings::_helpers::render::declaration_values::constant_declaration_sql; use crate::bindings::_helpers::render::macro_expansion_result::macro_expansion_classes; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -22,7 +23,6 @@ const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const VALIDATION_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.validation"; const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; -const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; const PROJECT_VARS_MODULE: &str = "sqlbuild._native"; const MACRO_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; @@ -60,19 +60,13 @@ fn loop_host<'py>(context: &Bound<'py, PyAny>) -> PyResult> { let context: &Bound<'_, PyAny> = context.bind(py); let passed: Bound<'_, PyDict> = keywords.map_or_else(|| PyDict::new(py), |keywords| keywords.clone()); - call_with( - &imported(py, DECLARATIONS_MODULE, "render_constant_declaration")?, - &[ - ("declaration", passed.as_any().get_item("declaration")?), - ("value_renderer", context.getattr("value_renderer")?), - ( - "collection_rendering", - context.getattr("collection_rendering")?, - ), - ("file_path", passed.as_any().get_item("file_path")?), - ], - ) - .map(Bound::unbind) + let file_path: Bound<'_, PyAny> = passed.as_any().get_item("file_path")?; + let rendered: String = constant_declaration_sql( + &passed.as_any().get_item("declaration")?, + &context.getattr("value_renderer")?, + (&context.getattr("collection_rendering")?, Some(&file_path)), + )?; + Ok(PyString::new(py, &rendered).into_any().unbind()) }; let keywords: Bound<'py, PyDict> = PyDict::new(py); keywords.set_item( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs index 5b67c33912..65482d6dc3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs @@ -6,10 +6,10 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::attachments::cursor_intrinsics::validated_model_cursor_sql; use crate::bindings::_helpers::compile_driver::assembly_project::ADAPTER_NAME_MODULE; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::render::declaration_values::declaration_scope_resolver; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; const NATIVE_MODULE: &str = "sqlbuild._native"; /// The `CompiledSqlScenario` fields copied from its input, in Python's keyword order. const SCENARIO_FIELDS: [&str; 12] = [ @@ -63,14 +63,10 @@ pub(crate) fn test_model_query_overrides<'py>( declaration_scope.getattr("resolver")?.getattr("lookup")? }; let loaded_macros: Bound<'py, PyAny> = inputs.getattr("loaded_macros")?; - let declaration_resolver: Bound<'py, PyAny> = call_with( - &imported(py, DECLARATIONS_MODULE, "build_declaration_scope_resolver")?, - &[ - ("discovered_inputs", inputs.getattr("discovered_inputs")?), - ("scope_index", inputs.getattr("scope_index")?), - ("loaded_macros", loaded_macros.clone()), - ("lookup", lookup), - ], + let declaration_resolver: Bound<'py, PyAny> = declaration_scope_resolver( + &inputs.getattr("discovered_inputs")?, + &inputs.getattr("scope_index")?, + (&loaded_macros, &lookup), )?; let expand: Bound<'py, PyAny> = py.import(NATIVE_MODULE)?.getattr("expand_sql_macros")?; for model_input in inputs.getattr("model_inputs")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs new file mode 100644 index 0000000000..05dfc3df4a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs @@ -0,0 +1,306 @@ +//! Authored SQL expanded through project vars, declarations and macros, with each pass's spans. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::model_config::template_data::record_reads; +use crate::bindings::_helpers::render::declaration_expansion::{ + DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, SpanRow, + expand_declarations, span_rows, +}; +use crate::bindings::_helpers::render::declaration_values::resolved_declaration_context; +use crate::bindings::_helpers::render::macro_expansion_result::expand_sql_macros; +use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; +use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, interpolation_batch}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; +const FUNCTOOLS_MODULE: &str = "functools"; +const DECLARATION_FIELDS: [&str; 4] = [ + "enums", + "constants", + "inaccessible_enums", + "inaccessible_constants", +]; + +/// `sql` with `@@` tokens interpolated and each substitution's span, raising its error. +fn substituted<'py>( + sql: &Bound<'py, PyAny>, + file_path: &Bound<'py, PyAny>, + (effective_vars, context_values): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult<(String, Vec)> { + let py: Python<'py> = sql.py(); + let sources = InterpolationSources( + effective_vars.downcast()?.clone(), + imported(py, UNICODE_ENVIRONMENT_MODULE, "UnicodeEnvironment")?.call0()?, + if context_values.is_none() { + None + } else { + Some(context_values.downcast()?.clone()) + }, + imported(py, NATIVE_MODULE, "render_project_var_text")?, + ); + let text: String = sql.extract()?; + let path: String = file_path.str()?.extract()?; + let rows = interpolation_batch(sources, &[(text.clone(), path)])?; + let Some((rendered, spans, reads, error)) = rows.into_iter().next() else { + return Ok((text, Vec::new())); + }; + record_reads(py, reads.iter().map(|(kind, name)| (*kind, name.as_str())))?; + if let Some(message) = error { + return Err(PyErr::from_value( + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?.call1((message,))?, + )); + } + Ok((rendered.unwrap_or(text), spans)) +} + +/// `ExpansionSpan` objects for span rows, in order. +fn span_objects<'py>(py: Python<'py>, spans: &[SpanRow]) -> PyResult> { + let span_class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS_MODULE, "ExpansionSpan")?; + let objects: Vec> = spans + .iter() + .map(|span| span_class.call1(*span)) + .collect::>()?; + PyTuple::new(py, objects) +} + +/// The authored SQL's public declaration context: visible and inaccessible enums and constants. +fn declaration_context<'py>( + py: Python<'py>, + declarations: &Bound<'py, PyAny>, +) -> PyResult> { + let fields: Vec<(&str, Bound<'py, PyAny>)> = DECLARATION_FIELDS + .iter() + .map(|field| { + Ok(( + *field, + if declarations.is_none() { + PyDict::new(py).into_any() + } else { + declarations.getattr(*field)? + }, + )) + }) + .collect::>()?; + call_with( + &imported(py, COMPILE_MODELS_MODULE, "DeclarationResolutionContext")?, + &fields, + ) +} + +/// `sql` with natively scanned enum and constant references spliced, and their spans. +fn declarations_expanded<'py>( + sql: &str, + file_path: &Bound<'py, PyAny>, + (declarations, value_renderer, collection_rendering): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult<(String, Vec)> { + let py: Python<'py> = file_path.py(); + let (references, stop) = declaration_scan(sql); + if references.is_empty() && stop.is_none() { + return Ok((sql.to_owned(), Vec::new())); + } + let context: Bound<'py, PyAny> = declaration_context(py, declarations)?; + let render_constant: Bound<'py, PyAny> = imported(py, FUNCTOOLS_MODULE, "partial")?.call( + (imported(py, NATIVE_MODULE, "render_constant_declaration")?,), + Some(&keywords( + py, + &[ + ("value_renderer", value_renderer.clone()), + ("collection_rendering", collection_rendering.clone()), + ("file_path", file_path.clone()), + ], + )?), + )?; + let classes = ExpansionClasses { + usage_record: imported(py, SCOPE_MODELS_MODULE, "UsageRecord")?, + compile_error: imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?, + }; + let path: String = file_path.str()?.extract()?; + let expanded: ExpandedDeclarations<'py> = expand_declarations(&DeclarationExpansion { + sql, + references: &references, + stop, + context: &context, + consumer: ExpansionConsumer { + identity: None, + relationship_visibility: false, + path: &path, + }, + render_constant: &render_constant, + classes: &classes, + })?; + let spans: Vec = span_rows(&expanded.spliced); + Ok((expanded.spliced.sql, spans)) +} + +/// Expand authored SQL, returning each pass's substitution spans in order. +fn authored_sql_with_spans<'py>(request: &Request<'_, 'py>) -> PyResult> { + let py: Python<'py> = request.py(); + let none = || py.None().into_bound(py); + let file_path: Bound<'py, PyAny> = request.get("file_path")?; + let (interpolated, interpolation_spans) = substituted( + &request.get("sql")?, + &file_path, + ( + &request.get("effective_vars")?, + &request.optional("context_values")?.unwrap_or_else(none), + ), + )?; + let (declared, declaration_spans) = declarations_expanded( + &interpolated, + &file_path, + ( + &request.optional("declarations")?.unwrap_or_else(none), + &request.get("value_renderer")?, + &request.get("collection_rendering")?, + ), + )?; + let macro_options: Bound<'py, PyDict> = keywords( + py, + &[ + ("sql", PyString::new(py, &declared).into_any()), + ("file_path", file_path), + ("loaded_macros", request.get("loaded_macros")?), + ("macro_context", request.get("macro_context")?), + ( + "declaration_resolver", + request + .optional("declaration_resolver")? + .unwrap_or_else(none), + ), + ], + )?; + let expansion: Bound<'py, PyAny> = expand_sql_macros(py, Some(macro_options))?; + let passes: Bound<'py, PyTuple> = PyTuple::new( + py, + [ + span_objects(py, &interpolation_spans)?.into_any(), + span_objects(py, &declaration_spans)?.into_any(), + expansion.getattr("spans")?, + ], + )?; + PyTuple::new(py, [expansion.getattr("sql")?, passes.into_any()]) +} + +/// Expand authored SQL, returning each pass's substitution spans in order. +#[pyfunction] +#[pyo3(signature = (**request))] +fn expand_authored_sql_with_spans<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + authored_sql_with_spans(&Request::new("expand_authored_sql_with_spans", request)?) +} + +/// The declarations visible to `file_path`: the resolver's, else the context's, plus model-local. +fn path_declarations<'py>( + context: &Bound<'py, PyAny>, + file_path: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = context.py(); + let resolution_class: Bound<'py, PyAny> = + imported(py, COMPILE_MODELS_MODULE, "DeclarationResolutionContext")?; + let resolver: Bound<'py, PyAny> = context.getattr("declaration_resolver")?; + let declarations: Bound<'py, PyAny> = if resolver.is_none() { + call_with( + &resolution_class, + &[ + ("enums", context.getattr("enums")?), + ("constants", context.getattr("constants")?), + ], + )? + } else { + resolved_declaration_context(&resolver, file_path, None)? + }; + let local: Bound<'py, PyAny> = context + .getattr("local_declarations")? + .call_method1("get", (file_path,))?; + if local.is_none() { + return Ok(declarations); + } + let merged = |field: &str| -> PyResult> { + declarations.getattr(field)?.bitor(local.getattr(field)?) + }; + call_with( + &resolution_class, + &[ + ("enums", merged("enums")?), + ("constants", merged("constants")?), + ( + "inaccessible_enums", + declarations.getattr("inaccessible_enums")?, + ), + ( + "inaccessible_constants", + declarations.getattr("inaccessible_constants")?, + ), + ], + ) +} + +/// Expand authored SQL with a project's expansion context and report each pass's spans. +#[pyfunction] +#[pyo3(signature = (*, sql, file_path, context))] +fn expand_sql_with_spans<'py>( + sql: &Bound<'py, PyAny>, + file_path: &Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = sql.py(); + let arguments: Bound<'py, PyDict> = keywords( + py, + &[ + ("sql", sql.clone()), + ("file_path", file_path.clone()), + ("effective_vars", context.getattr("effective_vars")?), + ("loaded_macros", context.getattr("loaded_macros")?), + ("macro_context", context.getattr("macro_context")?), + ("declarations", path_declarations(context, file_path)?), + ( + "declaration_resolver", + context.getattr("declaration_resolver")?, + ), + ("value_renderer", context.getattr("value_renderer")?), + ( + "collection_rendering", + context.getattr("collection_rendering")?, + ), + ], + )?; + authored_sql_with_spans(&Request::new("expand_sql_with_spans", Some(&arguments))?) +} + +/// Replace SQL interpolation tokens, returning the span of every substitution. +#[pyfunction] +#[pyo3(signature = (*, sql, file_path, effective_vars, context_values=None))] +fn substitute_sql_vars_with_spans<'py>( + sql: &Bound<'py, PyAny>, + file_path: &Bound<'py, PyAny>, + effective_vars: &Bound<'py, PyAny>, + context_values: Option>, +) -> PyResult<(String, Bound<'py, PyTuple>)> { + let py: Python<'py> = sql.py(); + let context_values: Bound<'py, PyAny> = + context_values.unwrap_or_else(|| py.None().into_bound(py)); + let (substituted_sql, spans) = substituted(sql, file_path, (effective_vars, &context_values))?; + Ok((substituted_sql, span_objects(py, &spans)?)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(expand_authored_sql_with_spans, module)?)?; + module.add_function(wrap_pyfunction!(expand_sql_with_spans, module)?)?; + module.add_function(wrap_pyfunction!(substitute_sql_vars_with_spans, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs index 8cfb32f4c2..599a082a4d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs @@ -1,8 +1,8 @@ //! Enum and constant references expanded natively, for model queries and every other caller. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyInt, PyTuple}; -use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::errors::DeclarationError; use sqlbuild_analysis::compiled_project::main::declaration_error_message::declaration_error_message; use sqlbuild_analysis::compiled_project::main::enum_member_literal::enum_member_literal; @@ -292,35 +292,3 @@ pub(crate) fn span_rows(spliced: &SplicedSql) -> Vec { }) .collect() } - -/// Expand one string's scanned references; `source` is `(sql, references, stop code)`. -#[pyfunction] -fn expand_declaration_references<'py>( - source: (String, Vec, Option), - context: Bound<'py, PyAny>, - consumer: (Option>, bool, String), - renderers: (Bound<'py, PyAny>, ExpansionClasses<'py>), -) -> PyResult<(String, Vec, Vec>)> { - let (sql, references, stop) = source; - let (identity, relationship_visibility, path) = consumer; - let (render_constant, classes) = renderers; - let expanded: ExpandedDeclarations<'py> = expand_declarations(&DeclarationExpansion { - sql: &sql, - references: &references, - stop, - context: &context, - consumer: ExpansionConsumer { - identity, - relationship_visibility, - path: &path, - }, - render_constant: &render_constant, - classes: &classes, - })?; - let spans: Vec = span_rows(&expanded.spliced); - Ok((expanded.spliced.sql, spans, expanded.usages)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(expand_declaration_references, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs new file mode 100644 index 0000000000..bc6133bf9b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs @@ -0,0 +1,315 @@ +//! Declaration scope resolvers, per-path declaration contexts and adapter-rendered constants. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyInt, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const SCOPE_LOOKUP_MODULE: &str = "sqlbuild.compiler.scopes.main.build_scope_lookup"; +const NATIVE_CONTEXTS_MODULE: &str = + "sqlbuild.compiler.model_loop.main._build_native_declaration_contexts"; +const SQL_VALUE_ERRORS_MODULE: &str = "sqlbuild.sql_values.exceptions"; +const TYPES_MODULE: &str = "types"; +const SCALAR_KINDS: [&str; 6] = ["string", "integer", "boolean", "float", "decimal", "null"]; +const COLLECTION_KINDS: [&str; 2] = ["list", "set"]; +const ARRAY_RENDERING: &str = "array"; +const MAX_SQL_VALUE_SIZE: usize = 1_000_000; + +/// `(kind, name, owner)` identity of one declaration, built as `DeclarationIdentity`. +fn declaration_identity<'py>( + classes: &(Bound<'py, PyAny>, Bound<'py, PyAny>), + kind: &str, + name: &Bound<'py, PyAny>, + owner: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let (identity_class, kinds) = classes; + let kind: Bound<'py, PyAny> = kinds.getattr(kind)?; + match owner { + Some(owner) => identity_class.call1((kind, name, owner)), + None => identity_class.call1((kind, name)), + } +} + +/// Each project declaration's identity mapped to its process-local value. +fn runtime_declarations<'py>( + discovered: &Bound<'py, PyAny>, + loaded_macros: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let classes = ( + imported(py, SCOPE_MODELS_MODULE, "DeclarationIdentity")?, + imported(py, SCOPE_TYPES_MODULE, "DeclarationKind")?, + ); + let declarations: Bound<'py, PyDict> = PyDict::new(py); + for (files, kind) in [("enum_files", "ENUM"), ("constant_files", "CONSTANT")] { + for file in discovered.getattr(files)?.try_iter()? { + for declaration in file?.getattr("declarations")?.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + let identity = + declaration_identity(&classes, kind, &declaration.getattr("name")?, None)?; + declarations.set_item(identity, declaration)?; + } + } + } + let resource_identity: Bound<'py, PyAny> = + imported(py, SCOPE_MODELS_MODULE, "ResourceIdentity")?; + let model_kind: Bound<'py, PyAny> = + imported(py, SCOPE_TYPES_MODULE, "ResourceKind")?.getattr("MODEL")?; + for model_file in discovered.getattr("model_files")?.try_iter()? { + let model_file: Bound<'py, PyAny> = model_file?; + let name: Bound<'py, PyAny> = model_file + .getattr("header_values")? + .call_method1("get", ("name",))?; + let name: Bound<'py, PyAny> = if name.is_instance_of::() && name.is_truthy()? { + name + } else { + model_file.getattr("relative_path")?.getattr("stem")? + }; + let owner: Bound<'py, PyAny> = resource_identity.call1((&model_kind, name))?; + for (attribute, kind) in [ + ("enum_declarations", "ENUM"), + ("constant_declarations", "CONSTANT"), + ] { + for declaration in model_file.getattr(attribute)?.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + let identity = declaration_identity( + &classes, + kind, + &declaration.getattr("name")?, + Some(&owner), + )?; + declarations.set_item(identity, declaration)?; + } + } + } + if !loaded_macros.is_none() { + for name_and_macro in loaded_macros.call_method0("values")?.try_iter()? { + let declared_macro: Bound<'py, PyAny> = name_and_macro?; + let identity = + declaration_identity(&classes, "MACRO", &declared_macro.getattr("name")?, None)?; + declarations.set_item(identity, declared_macro)?; + } + } + Ok(declarations) +} + +/// Resources whose declarations differ from their directory's: private owners and grantees. +fn resource_specific<'py>(scope_index: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = scope_index.py(); + let private: Bound<'py, PyAny> = + imported(py, SCOPE_TYPES_MODULE, "ScopeKind")?.getattr("PRIVATE")?; + let mut resources: Vec> = Vec::new(); + for declaration in scope_index.getattr("declarations")?.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + let owner: Bound<'py, PyAny> = declaration.getattr("identity")?.getattr("owner")?; + if declaration.getattr("scope")?.is(&private) && !owner.is_none() { + resources.push(owner); + } + } + for grant in scope_index.getattr("grants")?.try_iter()? { + resources.push(grant?.getattr("resource")?); + } + PyFrozenSet::new(py, resources) +} + +/// Pair the serializable static index with original process-local declaration values. +pub(crate) fn declaration_scope_resolver<'py>( + discovered: &Bound<'py, PyAny>, + scope_index: &Bound<'py, PyAny>, + (loaded_macros, lookup): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let declarations: Bound<'py, PyDict> = runtime_declarations(discovered, loaded_macros)?; + let lookup: Bound<'py, PyAny> = if lookup.is_none() { + call_with( + &imported(py, SCOPE_LOOKUP_MODULE, "build_scope_lookup")?, + &[("index", scope_index.clone())], + )? + } else { + lookup.clone() + }; + let projection: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "DeclarationRuntimeProjection")?, + &[( + "declarations", + imported(py, TYPES_MODULE, "MappingProxyType")?.call1((&declarations,))?, + )], + )?; + let native_contexts: Bound<'py, PyAny> = call_with( + &imported( + py, + NATIVE_CONTEXTS_MODULE, + "build_native_declaration_contexts", + )?, + &[ + ("lookup", lookup.clone()), + ("declarations", declarations.into_any()), + ], + )?; + call_with( + &imported(py, COMPILE_MODELS_MODULE, "DeclarationScopeResolver")?, + &[ + ("project_dir", discovered.getattr("project_dir")?), + ("lookup", lookup), + ("projection", projection), + ( + "resource_specific", + resource_specific(scope_index)?.into_any(), + ), + ("native_contexts", native_contexts), + ], + ) +} + +/// Canonical visibility for one authored path, projected onto runtime declaration values. +pub(crate) fn resolved_declaration_context<'py>( + resolver: &Bound<'py, PyAny>, + file_path: &Bound<'py, PyAny>, + resource: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let py: Python<'py> = resolver.py(); + let resource: Bound<'py, PyAny> = + resource.map_or_else(|| py.None().into_bound(py), Clone::clone); + resolver + .getattr("native_contexts")? + .call_method1("resolve", (resolver, file_path, resource))? + .get_item(0) +} + +/// The adapter's SQL for one constant under `selected` collection rendering, or its error. +fn rendered_constant<'py>( + declaration: &Bound<'py, PyAny>, + value_renderer: &Bound<'py, PyAny>, + selected: &Bound<'py, PyAny>, +) -> PyResult { + let value: Bound<'py, PyAny> = declaration.getattr("value")?; + let kind: String = value.getattr("kind")?.str()?.extract()?; + let method: &str = if SCALAR_KINDS.contains(&kind.as_str()) { + "render_typed_scalar" + } else if COLLECTION_KINDS.contains(&kind.as_str()) { + if selected.eq(ARRAY_RENDERING)? { + "render_typed_array" + } else { + "render_typed_value_list" + } + } else { + "render_typed_object" + }; + let rendered: String = + call_with(&value_renderer.getattr(method)?, &[("value", value)])?.extract()?; + let size: usize = rendered.len(); + if size > MAX_SQL_VALUE_SIZE { + let context: String = format!( + "{} constant '{}'", + declaration.getattr("relative_path")?.str()?, + declaration.getattr("name")?.str()? + ); + return Err(PyErr::from_value( + imported( + declaration.py(), + SQL_VALUE_ERRORS_MODULE, + "SqlValueValidationError", + )? + .call1((format!( + "{context} rendered value is {size} bytes; maximum is {MAX_SQL_VALUE_SIZE} bytes" + ),))?, + )); + } + Ok(rendered) +} + +/// Render one validated constant with the active adapter's typed-value contract. +pub(crate) fn constant_declaration_sql<'py>( + declaration: &Bound<'py, PyAny>, + value_renderer: &Bound<'py, PyAny>, + (collection_rendering, file_path): (&Bound<'py, PyAny>, Option<&Bound<'py, PyAny>>), +) -> PyResult { + let py: Python<'py> = declaration.py(); + let render_as: Bound<'py, PyAny> = declaration.getattr("render_as")?; + let selected: Bound<'py, PyAny> = if render_as.is_truthy()? { + render_as + } else { + collection_rendering.clone() + }; + let error: PyErr = match rendered_constant(declaration, value_renderer, &selected) { + Ok(rendered) => return Ok(rendered), + Err(error) => error, + }; + let rendering_error: Bound<'py, PyAny> = + imported(py, SQL_VALUE_ERRORS_MODULE, "SqlValueRenderingError")?; + let validation_error: Bound<'py, PyAny> = + imported(py, SQL_VALUE_ERRORS_MODULE, "SqlValueValidationError")?; + let raised: Bound<'py, PyAny> = error.value(py).clone().into_any(); + if !(raised.is_instance(&rendering_error)? || raised.is_instance(&validation_error)?) { + return Err(error); + } + let relative_path: Bound<'py, PyAny> = declaration.getattr("relative_path")?; + let location: Bound<'py, PyAny> = match file_path { + Some(file_path) if !file_path.is_none() => file_path.clone(), + _ => relative_path.clone(), + }; + let compile_error: Bound<'py, PyAny> = + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?.call1((format!( + "{} constant '{}' could not be rendered in '{}' by adapter '{}' as {}: {}", + relative_path.str()?, + declaration.getattr("name")?.str()?, + location.str()?, + value_renderer.getattr("adapter_name")?.str()?, + selected.getattr("value")?.str()?, + raised.str()?, + ),))?; + compile_error.setattr("__cause__", &raised)?; + Err(PyErr::from_value(compile_error)) +} + +/// Render one validated constant with the active adapter's typed-value contract. +#[pyfunction] +#[pyo3(signature = (*, declaration, value_renderer, collection_rendering, file_path=None))] +fn render_constant_declaration<'py>( + declaration: &Bound<'py, PyAny>, + value_renderer: &Bound<'py, PyAny>, + collection_rendering: &Bound<'py, PyAny>, + file_path: Option>, +) -> PyResult { + constant_declaration_sql( + declaration, + value_renderer, + (collection_rendering, file_path.as_ref()), + ) +} + +/// One enum member value as a SQL scalar literal. +#[pyfunction] +#[pyo3(signature = (*, value))] +fn render_enum_member_value(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_instance_of::() { + return value.str()?.extract(); + } + let text: String = value.extract()?; + Ok(format!("'{}'", text.replace('\'', "''"))) +} + +/// Canonical visibility for one authored path, projected onto runtime declaration values. +#[pyfunction] +#[pyo3(signature = (*, resolver, file_path, resource=None))] +fn resolve_declaration_context<'py>( + resolver: &Bound<'py, PyAny>, + file_path: &Bound<'py, PyAny>, + resource: Option>, +) -> PyResult> { + resolved_declaration_context(resolver, file_path, resource.as_ref()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_constant_declaration, module)?)?; + module.add_function(wrap_pyfunction!(render_enum_member_value, module)?)?; + module.add_function(wrap_pyfunction!(resolve_declaration_context, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index 1137cc93cb..d6fe88b2b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -9,6 +9,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; +use crate::bindings::_helpers::render::declaration_values::resolved_declaration_context; use crate::bindings::_helpers::render::macro_call_rows::span_row; use crate::bindings::_helpers::render::macro_expansion::{ ExpansionOutput, NativeMacroExpansion, expanded, @@ -24,7 +25,6 @@ const REF_MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; const REF_TYPES_MODULE: &str = "sqlbuild.python_nodes.types"; const MACRO_ARGUMENTS_MODULE: &str = "sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments"; -const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; const USER_MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; const ACTIVE_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; const BRIDGE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.macro_bridge.constants"; @@ -204,13 +204,9 @@ fn expansion_result<'py>(request: ExpansionRequest<'py>) -> PyResult = match (request.declarations, request.declaration_resolver) { (Some(declarations), _) => declarations, - (None, Some(resolver)) => call_with( - &imported(py, DECLARATIONS_MODULE, "resolve_declaration_context")?, - &[ - ("resolver", resolver), - ("file_path", request.file_path.clone()), - ], - )?, + (None, Some(resolver)) => { + resolved_declaration_context(&resolver, &request.file_path, None)? + } (None, None) => py.None().into_bound(py), }; let expansion: NativeMacroExpansion = NativeMacroExpansion::new( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs index 84bb516f80..f678671194 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs @@ -1,5 +1,7 @@ +pub(crate) mod authored_sql; pub(crate) mod declaration_expansion; pub(crate) mod declaration_indexes; +pub(crate) mod declaration_values; pub(crate) mod macro_arguments; pub(crate) mod macro_call_classes; pub(crate) mod macro_call_memo; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs index 2ea4348538..5f05957dfc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs @@ -3,12 +3,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::render::{ - declaration_expansion, declaration_indexes, macro_arguments, macro_calls, macro_expansion, - macro_expansion_result, macro_modules, + authored_sql, declaration_indexes, declaration_values, macro_arguments, macro_calls, + macro_expansion, macro_expansion_result, macro_modules, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - declaration_expansion::register(module)?; + authored_sql::register(module)?; + declaration_values::register(module)?; declaration_indexes::register(module)?; macro_arguments::register(module)?; macro_calls::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs index d516dcde60..f4eed0434b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs @@ -1,40 +1,16 @@ //! `@enum`/`@const` reference offsets scanned natively for the model loop. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_sqltext::compiler::main::declaration_references::scan_declaration_references; use sqlbuild_sqltext::compiler::models::{ DeclarationReference, DeclarationReferenceKind, DeclarationReferenceScan, DeclarationReferenceStop, }; -use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::types::CompilerDetach; - /// `(kind code, name, member, start, end)`: kind 0 is an enum member, 1 a constant. type ReferenceRow = (u8, String, Option, usize, usize); /// References and stop code: unclosed quote 0, block comment 1, invalid enum 2, constant 3. type ScanRow = (Vec, Option); -/// Return each SQL string's references and stopping error under this Python's `\\b`. -#[pyfunction] -fn scan_sql_declaration_references(py: Python<'_>, sqls: Vec) -> PyResult> { - scanned_rows(py, sqls, false) -} - -/// The same scan walking past malformed references, as Python's declaration usage walk did. -#[pyfunction] -fn scan_sql_declaration_usages(py: Python<'_>, sqls: Vec) -> PyResult> { - scanned_rows(py, sqls, true) -} - -fn scanned_rows(py: Python<'_>, sqls: Vec, skip_malformed: bool) -> PyResult> { - let scanned: Vec = py - .compiler_detach(|| Ok(scan_declaration_references(&sqls, skip_malformed))) - .map_err(compiler_error)?; - Ok(scanned.into_iter().map(scan_row).collect()) -} - /// One SQL string's references and stop code, scanned without walking past malformed ones. pub(crate) fn declaration_scan(sql: &str) -> ScanRow { let scanned: Vec = @@ -67,9 +43,3 @@ fn reference_row(reference: DeclarationReference) -> ReferenceRow { reference.end, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scan_sql_declaration_references, module)?)?; - module.add_function(wrap_pyfunction!(scan_sql_declaration_usages, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs index d94a7da68f..2b9fdf77b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs @@ -3,15 +3,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::sqltext::{ - code_scanner, declaration_references, model_headers, placeholders, sql_references, static_sql, + code_scanner, model_headers, placeholders, sql_references, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { model_headers::register(module)?; code_scanner::register(module)?; - static_sql::register(module)?; sql_references::register(module)?; - declaration_references::register(module)?; placeholders::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs index abb065732b..415807dfa7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs @@ -3,9 +3,9 @@ use std::cell::RefCell; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyString, PyStringMethods}; -use pyo3::{FromPyObject, PyErr, pyfunction, wrap_pyfunction}; +use pyo3::{FromPyObject, PyErr}; use sqlbuild_sqltext::compiler::main::sql_interpolation::interpolate_sql; use sqlbuild_sqltext::compiler::models::InterpolationRead; use sqlbuild_sqltext::compiler::types::InterpolationHost; @@ -142,15 +142,6 @@ fn key_text(key: &Bound<'_, PyAny>) -> String { key.str().map(|text| text.to_string()).unwrap_or_default() } -/// Interpolate each `(sql, file path)`; an error is returned in its row, not raised. -#[pyfunction] -fn interpolate_sql_batch<'py>( - sqls: Vec<(String, String)>, - sources: InterpolationSources<'py>, -) -> PyResult> { - interpolation_batch(sources, &sqls) -} - /// Each `(sql, file path)` interpolated in order; non-rendering Python failures raise. pub(crate) fn interpolation_batch( sources: InterpolationSources<'_>, @@ -211,8 +202,3 @@ fn read_rows(reads: Vec) -> Vec<(&'static str, String)> { }) .collect() } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(interpolate_sql_batch, module)?)?; - Ok(()) -} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index fa829070c1..f81b9a8964 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -93,6 +93,7 @@ from sqlbuild.compiler.compile.models import ( DeclarationScopeBuild, DeclarationScopeResolver, DynamicColumnContractProof, + ExpansionSpan, FunctionArgument, FunctionReturnColumn, InferredColumn, @@ -320,7 +321,7 @@ from sqlbuild.spec.contracts.types import ( WarehouseGroup, ) from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import StateSqlValueType +from sqlbuild.sql_values.types import CollectionRendering, StateSqlValueType BUILD_IDENTITY: str STAGE_CAPTURE_OMITTED_ATTRIBUTES: dict[str, frozenset[str]] @@ -2617,12 +2618,6 @@ def parse_schema_columns( def column_migration_problem( model_name: str, values: dict[str, object], columns: list[tuple[str, str | None]] ) -> NativeConfigError | None: ... -def expand_declaration_references( - source: tuple[str, list[tuple[int, str, str | None, int, int]], int | None], - context: object, - consumer: tuple[object | None, bool, str], - renderers: tuple[object, dict[str, object]], -) -> tuple[str, list[tuple[int, int, int, int]], list[Any]]: ... def expand_template_data( *, value: object, @@ -2718,23 +2713,6 @@ def resolve_cursor_intrinsics( *, sql: str, start_sql: str | None = None, end_sql: str | None = None ) -> tuple[str, bool]: ... def parse_macro_arguments(text: str, nested: list[tuple[int, int]]) -> tuple[object, ...]: ... -def scan_sql_declaration_references( - sqls: list[str], -) -> list[tuple[list[tuple[int, str, str | None, int, int]], int | None]]: ... -def scan_sql_declaration_usages( - sqls: list[str], -) -> list[tuple[list[tuple[int, str, str | None, int, int]], int | None]]: ... -def interpolate_sql_batch( - sqls: list[tuple[str, str]], - sources: tuple[ - Mapping[str, object], - Mapping[str, str], - Mapping[str, str | None] | None, - Callable[..., str], - ], -) -> list[ - tuple[str | None, list[tuple[int, int, int, int]], list[tuple[str, str]], str | None] -]: ... def scan_macro_call_sites( sql: str, ) -> tuple[ @@ -2760,6 +2738,43 @@ def fixture_projection_facts( sql: str, dialect: str | None, / ) -> tuple[list[tuple[str, str]], list[str], list[str]]: ... def find_macro_call_names(sql: str, /) -> tuple[str, ...]: ... +def render_constant_declaration( + *, + declaration: ConstantDeclaration, + value_renderer: TypedSqlValueRenderer, + collection_rendering: CollectionRendering, + file_path: Path | None = None, +) -> str: ... +def render_enum_member_value(*, value: str | int) -> str: ... +def resolve_declaration_context( + *, + resolver: DeclarationScopeResolver, + file_path: Path, + resource: ResourceIdentity | None = None, +) -> DeclarationResolutionContext: ... +def substitute_sql_vars_with_spans( + *, + sql: str, + file_path: Path, + effective_vars: dict[str, object], + context_values: Mapping[str, str | None] | None = None, +) -> tuple[str, tuple[ExpansionSpan, ...]]: ... +def expand_authored_sql_with_spans( + *, + sql: str, + file_path: Path, + effective_vars: dict[str, object], + loaded_macros: dict[str, LoadedMacro], + macro_context: MacroContext, + value_renderer: TypedSqlValueRenderer, + collection_rendering: CollectionRendering, + context_values: Mapping[str, str | None] | None = None, + declarations: DeclarationResolutionContext | None = None, + declaration_resolver: DeclarationScopeResolver | None = None, +) -> tuple[str, tuple[tuple[ExpansionSpan, ...], ...]]: ... +def expand_sql_with_spans( + *, sql: str, file_path: Path, context: SqlExpansionContext +) -> tuple[str, tuple[tuple[ExpansionSpan, ...], ...]]: ... def expand_sql_macros( *, sql: str, diff --git a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py deleted file mode 100644 index 8c1bfab4f3..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py +++ /dev/null @@ -1,222 +0,0 @@ -"""Compile-time resolution for enum and constant declarations.""" - -from __future__ import annotations - -from collections.abc import Mapping -from functools import partial -from pathlib import Path -from types import MappingProxyType - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - DeclarationExpansionResult, - DeclarationResolutionContext, - DeclarationRuntimeProjection, - DeclarationScopeResolver, - ExpansionSpan, - LoadedMacro, -) -from sqlbuild.compiler.compile.types import TypedSqlValueRenderer -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredConstantFile, - DiscoveredEnumFile, - DiscoveredProjectInputs, - DiscoveredSqlModelFile, - EnumDeclaration, -) -from sqlbuild.compiler.model_loop.main._build_native_declaration_contexts import ( - build_native_declaration_contexts, -) -from sqlbuild.compiler.model_loop.types import NativeDeclarationScan -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - ResourceIdentity, - ScopeIndex, - ScopeLookup, - UsageRecord, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - ResourceKind, - ScopeKind, -) -from sqlbuild.sql_values.exceptions import SqlValueRenderingError, SqlValueValidationError -from sqlbuild.sql_values.main.validate_rendered_size import validate_rendered_sql_value_size -from sqlbuild.sql_values.types import CollectionRendering, SqlValueKind - -_ACCEPTED_VALUES_AUDIT: str = "accepted_values" - - -def build_declaration_scope_resolver( - *, - discovered_inputs: DiscoveredProjectInputs, - scope_index: ScopeIndex, - loaded_macros: Mapping[str, LoadedMacro] | None = None, - lookup: ScopeLookup | None = None, -) -> DeclarationScopeResolver: - """Pair the serializable static index with original process-local declaration values.""" - - declarations: dict[ - DeclarationIdentity, EnumDeclaration | ConstantDeclaration | LoadedMacro - ] = {} - enum_file: DiscoveredEnumFile - for enum_file in discovered_inputs.enum_files: - for declaration in enum_file.declarations: - declarations[DeclarationIdentity(DeclarationKind.ENUM, declaration.name)] = declaration - constant_file: DiscoveredConstantFile - for constant_file in discovered_inputs.constant_files: - for declaration in constant_file.declarations: - declarations[DeclarationIdentity(DeclarationKind.CONSTANT, declaration.name)] = ( - declaration - ) - for model_file in discovered_inputs.model_files: - model_name_value: object = model_file.header_values.get("name") - model_name: str = ( - model_name_value - if isinstance(model_name_value, str) and model_name_value - else model_file.relative_path.stem - ) - owner: ResourceIdentity = ResourceIdentity(ResourceKind.MODEL, model_name) - for declaration in model_file.enum_declarations: - declarations[DeclarationIdentity(DeclarationKind.ENUM, declaration.name, owner)] = ( - declaration - ) - for declaration in model_file.constant_declarations: - declarations[DeclarationIdentity(DeclarationKind.CONSTANT, declaration.name, owner)] = ( - declaration - ) - if loaded_macros is not None: - for macro in loaded_macros.values(): - declarations[DeclarationIdentity(DeclarationKind.MACRO, macro.name)] = macro - scope_lookup: ScopeLookup = build_scope_lookup(index=scope_index) if lookup is None else lookup - return DeclarationScopeResolver( - project_dir=discovered_inputs.project_dir, - lookup=scope_lookup, - projection=DeclarationRuntimeProjection(declarations=MappingProxyType(declarations)), - resource_specific=frozenset( - declaration.identity.owner - for declaration in scope_index.declarations - if declaration.scope is ScopeKind.PRIVATE and declaration.identity.owner is not None - ) - | frozenset(grant.resource for grant in scope_index.grants), - native_contexts=build_native_declaration_contexts( - lookup=scope_lookup, declarations=declarations - ), - ) - - -def resolve_declaration_context( - *, - resolver: DeclarationScopeResolver, - file_path: Path, - resource: ResourceIdentity | None = None, -) -> DeclarationResolutionContext: - """Project canonical visibility for one authored path onto runtime declaration values.""" - - context: DeclarationResolutionContext - context, _ = resolver.native_contexts.resolve(resolver, file_path, resource) - return context - - -def build_model_declaration_indexes( - *, model_file: DiscoveredSqlModelFile -) -> tuple[dict[str, EnumDeclaration], dict[str, ConstantDeclaration]]: - """Build collision-checked declaration indexes private to one model.""" - - return ( - {declaration.name: declaration for declaration in model_file.enum_declarations}, - {declaration.name: declaration for declaration in model_file.constant_declarations}, - ) - - -def expand_scanned_declaration_references( # noqa: PLR0913 - *, - sql: str, - references: NativeDeclarationScan, - file_path: Path, - declarations: DeclarationResolutionContext, - value_renderer: TypedSqlValueRenderer, - collection_rendering: CollectionRendering, -) -> DeclarationExpansionResult: - """Splice natively scanned references and raise the first error the scan stopped at.""" - - scanned, stop = references - if not scanned and stop is None: - return DeclarationExpansionResult(sql=sql, spans=(), usages=()) - consumer: ResourceIdentity | DeclarationIdentity | None = declarations.consumer - expanded_sql, spans, usages = _native.expand_declaration_references( - (sql, list(scanned), stop), - declarations, - ( - consumer, - isinstance(consumer, ResourceIdentity) - and consumer.kind in {ResourceKind.TEST, ResourceKind.SCENARIO}, - str(file_path), - ), - ( - partial( - render_constant_declaration, - value_renderer=value_renderer, - collection_rendering=collection_rendering, - file_path=file_path, - ), - {"usage_record": UsageRecord, "compile_error": CompileInputError}, - ), - ) - return DeclarationExpansionResult( - sql=expanded_sql, - spans=tuple(ExpansionSpan(*span) for span in spans), - usages=tuple(dict.fromkeys(usages)), - ) - - -def render_constant_declaration( - *, - declaration: ConstantDeclaration, - value_renderer: TypedSqlValueRenderer, - collection_rendering: CollectionRendering, - file_path: Path | None = None, -) -> str: - """Render one validated constant with the active adapter's typed-value contract.""" - - selected_rendering: CollectionRendering = declaration.render_as or collection_rendering - try: - if declaration.value.kind in { - SqlValueKind.STRING, - SqlValueKind.INTEGER, - SqlValueKind.BOOLEAN, - SqlValueKind.FLOAT, - SqlValueKind.DECIMAL, - SqlValueKind.NULL, - }: - rendered: str = value_renderer.render_typed_scalar(value=declaration.value) - elif declaration.value.kind in {SqlValueKind.LIST, SqlValueKind.SET}: - rendered = ( - value_renderer.render_typed_array(value=declaration.value) - if selected_rendering == CollectionRendering.ARRAY - else value_renderer.render_typed_value_list(value=declaration.value) - ) - else: - rendered = value_renderer.render_typed_object(value=declaration.value) - validate_rendered_sql_value_size( - rendered_sql=rendered, - context=f"{declaration.relative_path} constant '{declaration.name}'", - ) - except (SqlValueRenderingError, SqlValueValidationError) as error: - raise CompileInputError( - f"{declaration.relative_path} constant '{declaration.name}' could not be rendered " - f"in '{file_path or declaration.relative_path}' by adapter " - f"'{value_renderer.adapter_name}' as " - f"{selected_rendering.value}: {error}", - ) from error - return rendered - - -def render_enum_member_value(*, value: str | int) -> str: - if isinstance(value, int): - return str(value) - escaped_value: str = value.replace("'", "''") - return f"'{escaped_value}'" diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py deleted file mode 100644 index a04a888ad3..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ /dev/null @@ -1,138 +0,0 @@ -"""SQL project variable and environment interpolation helpers.""" - -from __future__ import annotations - -from collections.abc import Mapping -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild._native import expand_sql_macros, record_template_reads, render_project_var_text -from sqlbuild.compiler.compile._helpers.render.declarations import ( - expand_scanned_declaration_references, -) -from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - DeclarationExpansionResult, - DeclarationResolutionContext, - DeclarationScopeResolver, - ExpansionSpan, - LoadedMacro, - MacroContext, - MacroExpansionResult, - SqlInterpolation, -) -from sqlbuild.compiler.compile.types import TypedSqlValueRenderer -from sqlbuild.compiler.model_loop.main._scan_native_declaration_references import ( - scan_native_declaration_references, -) -from sqlbuild.sql_values.types import CollectionRendering - -type _InterpolationRow = tuple[ - str | None, list[tuple[int, int, int, int]], list[tuple[str, str]], str | None -] - - -def expand_authored_sql_with_spans( - *, - sql: str, - file_path: Path, - effective_vars: dict[str, object], - loaded_macros: dict[str, LoadedMacro], - macro_context: MacroContext, - value_renderer: TypedSqlValueRenderer, - collection_rendering: CollectionRendering, - context_values: Mapping[str, str | None] | None = None, - declarations: DeclarationResolutionContext | None = None, - declaration_resolver: DeclarationScopeResolver | None = None, -) -> tuple[str, tuple[tuple[ExpansionSpan, ...], ...]]: - """Expand authored SQL, returning each pass's substitution spans in order.""" - - interpolated_sql: str - interpolation_spans: tuple[ExpansionSpan, ...] - interpolated_sql, interpolation_spans = substitute_sql_vars_with_spans( - sql=sql, - file_path=file_path, - effective_vars=effective_vars, - context_values=context_values, - ) - declaration_result: DeclarationExpansionResult = expand_scanned_declaration_references( - sql=interpolated_sql, - references=scan_native_declaration_references(sqls=(interpolated_sql,))[0], - file_path=file_path, - declarations=DeclarationResolutionContext( - enums=declarations.enums if declarations is not None else {}, - constants=declarations.constants if declarations is not None else {}, - inaccessible_enums=declarations.inaccessible_enums if declarations is not None else {}, - inaccessible_constants=( - declarations.inaccessible_constants if declarations is not None else {} - ), - ), - value_renderer=value_renderer, - collection_rendering=collection_rendering, - ) - declaration_expanded_sql: str = declaration_result.sql - declaration_spans: tuple[ExpansionSpan, ...] = declaration_result.spans - expansion: MacroExpansionResult = expand_sql_macros( - sql=declaration_expanded_sql, - file_path=file_path, - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_resolver=declaration_resolver, - ) - return expansion.sql, (interpolation_spans, declaration_spans, expansion.spans) - - -def substitute_sql_vars_with_spans( - *, - sql: str, - file_path: Path, - effective_vars: dict[str, object], - context_values: Mapping[str, str | None] | None = None, -) -> tuple[str, tuple[ExpansionSpan, ...]]: - """Replace SQL interpolation tokens, returning the span of every substitution.""" - - interpolation: SqlInterpolation = interpolate_sql_batch( - sqls=((sql, file_path),), effective_vars=effective_vars, context_values=context_values - )[0] - return applied_interpolation(interpolation), interpolation.spans - - -def interpolate_sql_batch( - *, - sqls: tuple[tuple[str, Path], ...], - effective_vars: dict[str, object], - context_values: Mapping[str, str | None] | None = None, -) -> tuple[SqlInterpolation, ...]: - """Interpolate every `(sql, file path)` natively; errors wait in their result.""" - - rows: list[_InterpolationRow] = _native.interpolate_sql_batch( - [(sql, str(file_path)) for sql, file_path in sqls], - (effective_vars, UnicodeEnvironment(), context_values, render_project_var_text), - ) - return tuple( - _interpolation(sql=sql, row=row) for (sql, _file_path), row in zip(sqls, rows, strict=True) - ) - - -def applied_interpolation(interpolation: SqlInterpolation) -> str: - """Record the interpolation's environment and context reads, then return or raise it.""" - - record_template_reads(interpolation.reads) - if interpolation.error is not None: - raise CompileInputError(interpolation.error) - return interpolation.sql - - -def _interpolation( - *, - sql: str, - row: _InterpolationRow, -) -> SqlInterpolation: - rendered, spans, reads, error = row - return SqlInterpolation( - sql=rendered if rendered is not None else sql, - spans=tuple(ExpansionSpan(*span) for span in spans), - reads=tuple(reads), - error=error, - ) diff --git a/src/sqlbuild/compiler/compile/main/expand_sql_with_spans.py b/src/sqlbuild/compiler/compile/main/expand_sql_with_spans.py deleted file mode 100644 index ec9f8da08e..0000000000 --- a/src/sqlbuild/compiler/compile/main/expand_sql_with_spans.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Expand one authored SQL body while reporting each substitution's span.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_context -from sqlbuild.compiler.compile._helpers.render.sql_vars import expand_authored_sql_with_spans -from sqlbuild.compiler.compile.models import ( - DeclarationResolutionContext, - ExpansionSpan, - SqlExpansionContext, -) - - -def expand_sql_with_spans( - *, sql: str, file_path: Path, context: SqlExpansionContext -) -> tuple[str, tuple[tuple[ExpansionSpan, ...], ...]]: - """Expand authored SQL and return each expansion pass's substitution spans.""" - - local_declarations: DeclarationResolutionContext | None = context.local_declarations.get( - file_path - ) - declarations: DeclarationResolutionContext = DeclarationResolutionContext( - enums=context.enums, - constants=context.constants, - ) - if context.declaration_resolver is not None: - declarations = resolve_declaration_context( - resolver=context.declaration_resolver, file_path=file_path - ) - if local_declarations is not None: - declarations = DeclarationResolutionContext( - enums=declarations.enums | local_declarations.enums, - constants=declarations.constants | local_declarations.constants, - inaccessible_enums=declarations.inaccessible_enums, - inaccessible_constants=declarations.inaccessible_constants, - ) - return expand_authored_sql_with_spans( - sql=sql, - file_path=file_path, - effective_vars=context.effective_vars, - loaded_macros=context.loaded_macros, - macro_context=context.macro_context, - declarations=declarations, - declaration_resolver=context.declaration_resolver, - value_renderer=context.value_renderer, - collection_rendering=context.collection_rendering, - ) diff --git a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py index 1978725d64..1a4727e3a3 100644 --- a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py +++ b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py @@ -11,9 +11,6 @@ discover_project_inputs, rebind_declaration_scope, ) -from sqlbuild.compiler.compile._helpers.render.declarations import ( - build_model_declaration_indexes, -) from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.models import ( ConstantDeclaration, @@ -68,12 +65,11 @@ def build_sql_expansion_context( enums, constants = _native.public_declaration_indexes(effective_discovered_inputs) local_declarations: dict[Path, DeclarationResolutionContext] = {} for model_file in effective_discovered_inputs.model_files: - local_enums: dict[str, EnumDeclaration] - local_constants: dict[str, ConstantDeclaration] - local_enums, local_constants = build_model_declaration_indexes(model_file=model_file) local_declarations[model_file.file_path] = DeclarationResolutionContext( - enums=local_enums, - constants=local_constants, + enums={declaration.name: declaration for declaration in model_file.enum_declarations}, + constants={ + declaration.name: declaration for declaration in model_file.constant_declarations + }, ) return SqlExpansionContext( effective_vars=effective_vars, diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 59439b520e..d219e66130 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -343,9 +343,7 @@ def render_constant(self, name: str) -> str: raise CompileInputError( "Constant SQL rendering requires a declaration resolved for this macro invocation" ) - from sqlbuild.compiler.compile._helpers.render.declarations import ( - render_constant_declaration, - ) + from sqlbuild._native import render_constant_declaration return render_constant_declaration( declaration=declaration, @@ -357,9 +355,7 @@ def render_enum_member(self, *, enum_name: str, member_name: str) -> str: """Render one visible enum member as a SQL scalar literal.""" value: str | int = self.enums[enum_name][member_name] - from sqlbuild.compiler.compile._helpers.render.declarations import ( - render_enum_member_value, - ) + from sqlbuild._native import render_enum_member_value return render_enum_member_value(value=value) diff --git a/src/sqlbuild/compiler/model_loop/main/_scan_native_declaration_references.py b/src/sqlbuild/compiler/model_loop/main/_scan_native_declaration_references.py deleted file mode 100644 index 89ca8425a5..0000000000 --- a/src/sqlbuild/compiler/model_loop/main/_scan_native_declaration_references.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Scan `@enum`/`@const` references of many SQL strings in one native call.""" - -from __future__ import annotations - -from collections.abc import Callable - -import sqlbuild._native as _native -from sqlbuild.compiler.model_loop.types import NativeDeclarationScan - - -def scan_native_declaration_references( - *, sqls: tuple[str, ...], skip_malformed: bool = False -) -> tuple[NativeDeclarationScan, ...]: - """Return each string's references and stop code; `skip_malformed` walks past malformed ones.""" - - scan_references: Callable[ - [list[str]], - list[tuple[list[tuple[int, str, str | None, int, int]], int | None]], - ] = ( - _native.scan_sql_declaration_usages - if skip_malformed - else _native.scan_sql_declaration_references - ) - return tuple((tuple(scan[0]), scan[1]) for scan in scan_references(list(sqls))) diff --git a/src/sqlbuild/compiler/model_loop/types.py b/src/sqlbuild/compiler/model_loop/types.py deleted file mode 100644 index 703716e74c..0000000000 --- a/src/sqlbuild/compiler/model_loop/types.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Row shapes the native model loop exchanges with its Rust bindings.""" - -type NativeDeclarationReference = tuple[int, str, str | None, int, int] -type NativeDeclarationScan = tuple[tuple[NativeDeclarationReference, ...], int | None] diff --git a/src/sqlbuild/lint/_helpers/expansion.py b/src/sqlbuild/lint/_helpers/expansion.py index 08429e283c..1f409d2c2a 100644 --- a/src/sqlbuild/lint/_helpers/expansion.py +++ b/src/sqlbuild/lint/_helpers/expansion.py @@ -7,12 +7,15 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import discover_project_inputs, external_identifiers_for_headers +from sqlbuild._native import ( + discover_project_inputs, + expand_sql_with_spans, + external_identifiers_for_headers, +) from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter from sqlbuild.compiler.compile.constants import MACRO_TOKEN from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main.expand_sql_with_spans import expand_sql_with_spans from sqlbuild.compiler.compile.main.sql_expansion_context import build_sql_expansion_context from sqlbuild.compiler.compile.models import ( CompiledSqlExpansion, diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index fd191cfe21..222e8961d4 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -16,9 +16,9 @@ from sqlbuild._native import ( build_declaration_scope, discover_project_inputs, + resolve_declaration_context, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_context from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 972ab91505..230ed782a7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -45,16 +45,6 @@ class SqlReferenceExtractionErrorTestCase: expected_help: str | None = None -@dataclass(frozen=True) -class StaticProjectVarBatchTestCase: - """Model SQL texts and the `(sql, error)` each interpolates to.""" - - description: str - sqls: tuple[str, ...] - effective_vars: dict[str, object] - expected_results: tuple[tuple[str, str | None], ...] - - @dataclass(frozen=True) class AuditFactoryAttachmentTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py index 4796b9a35e..d8b5f93900 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py @@ -6,8 +6,8 @@ import pytest +from sqlbuild._native import expand_authored_sql_with_spans from sqlbuild.compiler.compile._helpers.render.spans import map_through_passes -from sqlbuild.compiler.compile._helpers.render.sql_vars import expand_authored_sql_with_spans from sqlbuild.compiler.compile.models import ExpansionSpan, LoadedMacro, MacroContext, MappedOffset from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ExpandWithSpansTestCase from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py index 39898fea7e..d070fba082 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py @@ -6,14 +6,9 @@ import pytest -from sqlbuild.compiler.compile._helpers.render.sql_vars import ( - interpolate_sql_batch, - substitute_sql_vars_with_spans, -) +from sqlbuild._native import substitute_sql_vars_with_spans from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import SqlInterpolation from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( - StaticProjectVarBatchTestCase, SubstituteSqlVarsErrorTestCase, SubstituteSqlVarsTestCase, ) @@ -24,6 +19,18 @@ @pytest.mark.parametrize( "test_case", [ + SubstituteSqlVarsTestCase( + description="skips dollar-quoted text and positional parameters", + sql="SELECT $tag$ it's $$ -- @@status $tag$, price$1$, $1, @@revision", + effective_vars={"revision": 7, "status": "ready"}, + expected_sql="SELECT $tag$ it's $$ -- ready $tag$, price$1$, $1, 7", + ), + SubstituteSqlVarsTestCase( + description="keeps deferred placeholders and commented tokens", + sql="-- @@revision\nSELECT @@revision, '@@status', @@@window_start", + effective_vars={"revision": 7, "status": "ready"}, + expected_sql="-- @@revision\nSELECT 7, 'ready', @@@window_start", + ), SubstituteSqlVarsTestCase( description="replaces project var in SQL expression", sql="SELECT * FROM orders WHERE region = @@target_region", @@ -223,48 +230,5 @@ def test_given_missing_var_when_substituting_then_raises( ) -@pytest.mark.parametrize( - "test_case", - [ - StaticProjectVarBatchTestCase( - description="each model is interpolated; an error waits in its own result", - sqls=( - "SELECT @@revision, '@@status', @@@window_start", - "-- @@revision\nSELECT 1", - "SELECT '@@ENV:SQLBUILD_TEST_BATCH_UNSET'", - "SELECT $tag$ it's $$ -- @@status $tag$, price$1$, $1, @@revision", - "SELECT @@grants", - ), - effective_vars={"revision": 7, "status": "ready", "grants": {"role": "analyst"}}, - expected_results=( - ("SELECT 7, 'ready', @@@window_start", None), - ("-- @@revision\nSELECT 1", None), - ( - "SELECT '@@ENV:SQLBUILD_TEST_BATCH_UNSET'", - "unknown environment variable '@@ENV:SQLBUILD_TEST_BATCH_UNSET' in " - "'models/test_model.sql'", - ), - ("SELECT $tag$ it's $$ -- ready $tag$, price$1$, $1, 7", None), - ( - "SELECT @@grants", - "SQL variable '@@grants' is an object and cannot be interpolated as text: " - '{"role":"analyst"}. Use a macro to consume structured vars.', - ), - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_model_sqls_when_interpolating_a_batch_then_each_holds_its_sql_or_error( - test_case: StaticProjectVarBatchTestCase, -) -> None: - results: tuple[SqlInterpolation, ...] = interpolate_sql_batch( - sqls=tuple((sql, _FILE_PATH) for sql in test_case.sqls), - effective_vars=test_case.effective_vars, - ) - - assert [(result.sql, result.error) for result in results] == list(test_case.expected_results) - - if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py index 411ca2e86f..7fabbf9cad 100644 --- a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py @@ -7,8 +7,7 @@ import pytest -from sqlbuild._native import run_lint -from sqlbuild.compiler.compile.main.expand_sql_with_spans import expand_sql_with_spans +from sqlbuild._native import expand_sql_with_spans, run_lint from sqlbuild.compiler.compile.main.map_expanded_offset import map_expanded_offset from sqlbuild.compiler.compile.models import MappedOffset, SqlExpansionContext from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs From 9be203cfd47c3f49f60db7fc485985f9d555c7a6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 03:43:46 +0100 Subject: [PATCH 09/40] refactor: build the project assembly request natively The native assemble_native_project_resources binding reads the compile inputs, renders opaque vars, looks up template environment names and materialises NativeProjectResources itself. The Python request and resource helpers, their constants and row types, and the reviewed handler entry for the removed opaque var renderer go. --- .../bindings/_helpers/authored_values/mod.rs | 2 +- .../_helpers/authored_values/project_vars.rs | 5 + .../bindings/_helpers/project_assembly/mod.rs | 1 + .../project_assembly/native_resources.rs | 479 ++++++++++++++++++ .../_helpers/project_assembly/registration.rs | 3 +- .../_helpers/project_assembly/resources.rs | 312 +----------- python-boundary.toml | 4 - src/sqlbuild/_native.pyi | 40 +- .../_helpers/native_stages/assembly.py | 4 +- .../project_assembly/_helpers/__init__.py | 1 - .../project_assembly/_helpers/request.py | 157 ------ .../project_assembly/_helpers/resources.py | 72 --- .../compiler/project_assembly/constants.py | 21 - .../_assemble_native_project_resources.py | 25 - .../compiler/project_assembly/types.py | 33 -- .../compiler/project_assembly/_test_types.py | 11 - .../compiler/project_assembly/helpers.py | 15 - .../test_native_project_assembly.py | 65 --- 18 files changed, 497 insertions(+), 753 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs delete mode 100644 src/sqlbuild/compiler/project_assembly/_helpers/__init__.py delete mode 100644 src/sqlbuild/compiler/project_assembly/_helpers/request.py delete mode 100644 src/sqlbuild/compiler/project_assembly/_helpers/resources.py delete mode 100644 src/sqlbuild/compiler/project_assembly/constants.py delete mode 100644 src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs index 282e72911a..8e07eaa020 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs @@ -1,3 +1,3 @@ mod fields; -mod project_vars; +pub(crate) mod project_vars; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs index cd6f3e1c86..9ecb1ae789 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs @@ -49,6 +49,11 @@ fn preview_json(value: &Bound<'_, PyAny>) -> PyResult { #[pyfunction] #[pyo3(signature = (*, value, label))] fn render_project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { + project_var_text(value, label) +} + +/// One project var value as interpolated text; structured values are rejected. +pub(crate) fn project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { let py: Python<'_> = value.py(); if value.is_none() { return Ok(String::new()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs index 5f1224cc0d..b58d1f0cc0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod declared_relations; +pub(crate) mod native_resources; pub(crate) mod registration; pub(crate) mod resources; pub(crate) mod syntax_checks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs new file mode 100644 index 0000000000..bc35313996 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs @@ -0,0 +1,479 @@ +//! The native project assembly request built from compile inputs, and its resource facts. + +use pyo3::exceptions::{PyException, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyBoolMethods, PyInt, PyList, PyString, PyStringMethods, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; +use sqlbuild_analysis::assembly::project::errors::AssemblyError; +use sqlbuild_analysis::assembly::project::main::assemble_project_resources::assemble_project_resources; +use sqlbuild_analysis::assembly::project::main::environment_template_names::environment_template_names; +use sqlbuild_analysis::assembly::project::models::{ + AuditFacts, FunctionFacts, InputRead, ModelFacts, Namespace, OpaqueRender, OpaqueVariable, + ProjectRequest, ProjectResources, Reference, ResourceFailure, ResourceSlot, SeedDefaults, + SeedFacts, SourceFacts, SyntaxCheck, TargetNamespace, Variable, +}; +use sqlbuild_core::panics::main::native_failure::native_failure; +use sqlbuild_model_config::templates::models::Scalar; + +use crate::bindings::_helpers::authored_values::project_vars::project_var_text; +use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_error}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::model_config::template_data::record_reads; +use crate::bindings::types::CompilerDetach; + +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const ASSEMBLY_MODELS_MODULE: &str = "sqlbuild.compiler.project_assembly.models"; +const OS_MODULE: &str = "os"; +const PROJECT_ASSEMBLY_CONTEXT: &str = "native project assembly"; +const GENERIC_DIALECT: &str = "generic"; +const RENDER_LABEL: &str = "\u{0}"; +const SOURCE_SLOT: &str = "source"; +const SEED_SLOT: &str = "seed"; +const AUDIT_SLOT: &str = "audit"; +const ENVIRONMENT_READ: &str = "env"; +const CONTEXT_READ: &str = "ctx"; +const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; + +/// `(kind, name, package)`. +type ReferenceRow = (String, String, Option); +/// `(name, managed, database, schema)`. +type SourceRow = (String, bool, Option, Option); +/// `(kind, name, tags, references)`. +type FunctionRow = (String, String, Vec, Vec); +/// `(references, attached (kind, name))`. +type AuditRow = (Vec, Option<(String, String)>); + +/// How one opaque variable renders as text, or the Python exception rendering it raises. +enum Render<'py> { + Text(String), + Rejected(String), + Raises(Bound<'py, PyAny>), +} + +/// The plain scalar `value` crosses as: `None`, exact `bool`, `str` or 64-bit `int`. +fn plain_scalar(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(Some(Scalar::Null)); + } + if let Ok(flag) = value.downcast_exact::() { + return Ok(Some(Scalar::Bool(flag.is_true()))); + } + if let Ok(text) = value.downcast_exact::() { + return Ok(Some(Scalar::Text(text.to_str()?.to_owned()))); + } + if value.is_exact_instance_of::() { + return Ok(match value.extract::() { + Ok(number) => Some(Scalar::Text(number.to_string())), + Err(_) => None, + }); + } + Ok(None) +} + +/// `render_project_var_text` under a label no message starts with otherwise. +fn opaque_render<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + match project_var_text(value, RENDER_LABEL) { + Ok(text) => Ok(Render::Text(text)), + Err(error) if !error.is_instance_of::(py) => Err(error), + Err(error) => { + let raised: Bound<'py, PyAny> = error.value(py).clone().into_any(); + if error.is_instance_of::(py) { + let message: String = raised.str()?.extract()?; + if let Some(rest) = message.strip_prefix(RENDER_LABEL) { + return Ok(Render::Rejected(rest.to_owned())); + } + } + Ok(Render::Raises(raised)) + } + } +} + +/// Each effective var as the native request reads it, and the exceptions opaque ones raise. +fn variables<'py>( + effective_vars: &Bound<'py, PyAny>, +) -> PyResult<(Vec<(String, Variable)>, Vec>)> { + let mut rows: Vec<(String, Variable)> = Vec::new(); + let mut held: Vec> = Vec::new(); + for item in effective_vars.call_method0("items")?.try_iter()? { + let (name, value): (String, Bound<'py, PyAny>) = item?.extract()?; + if let Some(scalar) = plain_scalar(&value)? { + rows.push((name, Variable::Scalar(scalar))); + continue; + } + let render: OpaqueRender = match opaque_render(&value)? { + Render::Text(text) => OpaqueRender::Text(text), + Render::Rejected(message) => OpaqueRender::Rejected(message), + Render::Raises(error) => { + held.push(error); + OpaqueRender::Raises(held.len() - 1) + } + }; + let text: String = value.str()?.extract::()?; + rows.push((name, Variable::Opaque(OpaqueVariable { text, render }))); + } + Ok((rows, held)) +} + +fn optional_text(value: &Bound<'_, PyAny>, attribute: &str) -> PyResult> { + value.getattr(attribute)?.extract() +} + +/// Python's own `os.environ` lookup of every name an `ENV:` template may read. +fn environment_rows( + py: Python<'_>, + values: &[Option], +) -> PyResult)>> { + let environment: Bound<'_, PyAny> = imported(py, OS_MODULE, "environ")?; + environment_template_names(values) + .into_iter() + .map(|name| { + let value: Option = environment.call_method1("get", (&name,))?.extract()?; + Ok((name, value)) + }) + .collect() +} + +fn references(rows: Vec) -> Vec { + rows.into_iter() + .map(|(kind, name, package)| Reference { + kind, + name, + package, + }) + .collect() +} + +/// Model config tags as the project graph indexes them. +fn tags(value: &Bound<'_, PyAny>) -> PyResult> { + if !(value.is_instance_of::() || value.is_instance_of::()) { + return Ok(Vec::new()); + } + value + .try_iter()? + .map(|item| Ok(item?.str()?.extract::()?)) + .collect() +} + +fn model_facts(model_input: &Bound<'_, PyAny>, checks: &Bound<'_, PyAny>) -> PyResult { + let model_file: Bound<'_, PyAny> = model_input.getattr("model_file")?; + let mut reference_rows: Vec = Vec::new(); + for reference in model_input.getattr("references")?.try_iter()? { + let reference: Bound<'_, PyAny> = reference?; + reference_rows.push(( + reference.getattr("ref_kind")?.str()?.extract::()?, + reference.getattr("ref_name")?.extract()?, + reference.getattr("ref_package")?.extract()?, + )); + } + let mut syntax_checks: Vec = Vec::new(); + for check in checks.try_iter()? { + let (sql, placeholders): (String, Bound<'_, PyAny>) = check?.extract()?; + let placeholders: Pairs = if placeholders.is_truthy()? { + placeholders + .call_method0("items")? + .try_iter()? + .map(|pair| pair?.extract()) + .collect::>()? + } else { + Vec::new() + }; + syntax_checks.push(SyntaxCheck { sql, placeholders }); + } + Ok(ModelFacts { + name: model_file + .getattr("file_path")? + .getattr("stem")? + .extract()?, + directory: model_file + .getattr("relative_path")? + .getattr("parent")? + .str()? + .to_string(), + tags: tags( + &model_input + .getattr("config")? + .getattr("values")? + .call_method1("get", ("tags",))?, + )?, + references: references(reference_rows), + syntax_checks, + }) +} + +/// Every item of `inputs.`. +fn items<'py>(inputs: &Bound<'py, PyAny>, attribute: &str) -> PyResult>> { + inputs.getattr(attribute)?.try_iter()?.collect() +} + +/// The native assembly request for compile inputs, and the exceptions opaque vars raise. +fn project_request<'py>( + inputs: &Bound<'py, PyAny>, + dialect: Option, + syntax_checks: &Bound<'py, PyAny>, +) -> PyResult<(ProjectRequest, Vec>)> { + let py: Python<'py> = inputs.py(); + let (variables, held) = variables(&inputs.getattr("effective_vars")?)?; + let target: Bound<'py, PyAny> = inputs.getattr("effective_target")?; + let target: Option = if target.is_none() { + None + } else { + Some(TargetNamespace { + database: optional_text(&target, "database")?, + schema: optional_text(&target, "schema")?, + loader_schema: optional_text(&target, "loader_schema")?, + }) + }; + let defaults: Bound<'py, PyAny> = inputs.getattr("project_config")?.getattr("defaults")?; + let defaults = SeedDefaults { + database: optional_text(&defaults, "database")?, + schema: optional_text(&defaults, "schema")?, + seed_database: optional_text(&defaults, "seed_database")?, + seed_schema: optional_text(&defaults, "seed_schema")?, + }; + let seed_inputs: Vec> = items(inputs, "seed_inputs")?; + let mut seeds: Vec = Vec::with_capacity(seed_inputs.len()); + for seed_input in &seed_inputs { + let entry: Bound<'py, PyAny> = seed_input.getattr("schema_entry")?; + seeds.push(SeedFacts { + name: entry.getattr("name")?.extract()?, + tags: entry + .getattr("tags")? + .try_iter()? + .map(|tag| tag?.extract()) + .collect::>()?, + database: optional_text(&entry, "database")?, + schema: optional_text(&entry, "schema")?, + }); + } + let mut template_values: Vec> = Vec::new(); + if let Some(target) = &target { + template_values.extend([ + target.database.clone(), + target.schema.clone(), + target.loader_schema.clone(), + ]); + } + template_values.extend([ + defaults.database.clone(), + defaults.schema.clone(), + defaults.seed_database.clone(), + defaults.seed_schema.clone(), + ]); + for seed in &seeds { + template_values.extend([seed.database.clone(), seed.schema.clone()]); + } + let model_inputs: Vec> = items(inputs, "model_inputs")?; + let checks: Vec> = syntax_checks.try_iter()?.collect::>()?; + if checks.len() != model_inputs.len() { + return Err(PyValueError::new_err( + "zip() arguments have different lengths: model inputs and syntax checks", + )); + } + let models: Vec = model_inputs + .iter() + .zip(&checks) + .map(|(model_input, checks)| model_facts(model_input, checks)) + .collect::>()?; + let rows = |attribute: &str| -> PyResult>> { + items(inputs, attribute)? + .into_iter() + .map(|input| input.getattr("assembly_row")) + .collect() + }; + let sources: Vec = rows("source_inputs")? + .into_iter() + .map(|row| { + let (name, managed, database, schema): SourceRow = row.extract()?; + Ok(SourceFacts { + name, + managed, + database, + schema, + }) + }) + .collect::>()?; + let functions: Vec = rows("sql_function_inputs")? + .into_iter() + .map(|row| { + let (kind, name, tags, reference_rows): FunctionRow = row.extract()?; + Ok(FunctionFacts { + kind, + name, + tags, + references: references(reference_rows), + }) + }) + .collect::>()?; + let audits: Vec = rows("audit_inputs")? + .into_iter() + .map(|row| { + let (reference_rows, attached): AuditRow = row.extract()?; + Ok(AuditFacts { + references: references(reference_rows), + attached, + }) + }) + .collect::>()?; + let request = ProjectRequest { + dialect: dialect + .filter(|dialect| !dialect.is_empty()) + .unwrap_or_else(|| GENERIC_DIALECT.to_owned()), + target, + defaults, + variables, + environment: environment_rows(py, &template_values)?, + models, + sources, + seeds, + functions, + audits, + }; + Ok((request, held)) +} + +/// The exception Python's assembly raises at the failing resource. +fn failure_error<'py>( + py: Python<'py>, + error: AssemblyError, + held: &[Bound<'py, PyAny>], +) -> PyResult> { + match error { + AssemblyError::CompileInput { message, help } => { + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?.call( + (message,), + Some(&keywords( + py, + &[("help", help.into_pyobject(py)?.into_any())], + )?), + ) + } + AssemblyError::Value(message) => { + Ok(PyValueError::new_err(message).value(py).clone().into_any()) + } + AssemblyError::Raise(index) => match held.get(index) { + Some(error) => Ok(error.clone()), + None => Ok(NativeCompilerError::new_err(format!( + "{NATIVE_FAILURE_PREFIX}{PROJECT_ASSEMBLY_CONTEXT}: unknown failure kind 'held'" + )) + .value(py) + .clone() + .into_any()), + }, + AssemblyError::Internal(reason) => Err(compiler_error(native_failure( + PROJECT_ASSEMBLY_CONTEXT, + &reason, + ))), + } +} + +fn failure_object<'py>( + py: Python<'py>, + failure: ResourceFailure, + held: &[Bound<'py, PyAny>], +) -> PyResult> { + let slot: &str = match failure.slot { + ResourceSlot::Source => SOURCE_SLOT, + ResourceSlot::Seed => SEED_SLOT, + ResourceSlot::Audit => AUDIT_SLOT, + }; + call_with( + &imported(py, ASSEMBLY_MODELS_MODULE, "NativeResourceFailure")?, + &[ + ("slot", PyString::new(py, slot).into_any()), + ("index", failure.index.into_pyobject(py)?.into_any()), + ("error", failure_error(py, failure.error, held)?), + ], + ) +} + +fn namespace_row( + namespace: Namespace, +) -> ( + Option, + Option, + Option, + Option, +) { + ( + namespace.database, + namespace.schema, + namespace.logical_database, + namespace.logical_schema, + ) +} + +/// `NativeProjectResources` for the native result, replaying its template reads first. +fn resources_object<'py>( + py: Python<'py>, + resources: ProjectResources, + held: &[Bound<'py, PyAny>], +) -> PyResult> { + record_reads( + py, + resources.reads.iter().map(|read| match read { + InputRead::Environment(name) => (ENVIRONMENT_READ, name.as_str()), + InputRead::Context(name) => (CONTEXT_READ, name.as_str()), + }), + )?; + let tuple = |values: Bound<'py, PyAny>| -> PyResult> { + Ok(PyTuple::new(py, values.try_iter()?.collect::>>()?)?.into_any()) + }; + let failure: Bound<'py, PyAny> = match resources.failure { + Some(failure) => failure_object(py, failure, held)?, + None => py.None().into_bound(py), + }; + let seeds: Vec<_> = resources.seeds.into_iter().map(namespace_row).collect(); + call_with( + &imported(py, ASSEMBLY_MODELS_MODULE, "NativeProjectResources")?, + &[ + ( + "model_deps", + tuple(resources.model_deps.into_pyobject(py)?.into_any())?, + ), + ( + "model_syntax_valid", + tuple(resources.model_syntax_valid.into_pyobject(py)?.into_any())?, + ), + ( + "source_namespaces", + tuple(resources.sources.into_pyobject(py)?.into_any())?, + ), + ( + "seed_namespaces", + tuple(seeds.into_pyobject(py)?.into_any())?, + ), + ( + "function_deps", + tuple(resources.function_deps.into_pyobject(py)?.into_any())?, + ), + ( + "audit_scope_deps", + tuple(resources.audit_deps.into_pyobject(py)?.into_any())?, + ), + ("failure", failure), + ], + ) +} + +/// Return the resource facts and the resource Python's assembly raises at, if any. +#[pyfunction] +#[pyo3(signature = (*, inputs, dialect, syntax_checks))] +fn assemble_native_project_resources<'py>( + inputs: &Bound<'py, PyAny>, + dialect: Option, + syntax_checks: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let (request, held) = project_request(inputs, dialect, syntax_checks)?; + let resources: ProjectResources = py + .compiler_detach(|| assemble_project_resources(&request)) + .map_err(|reason| compiler_error(native_failure(PROJECT_ASSEMBLY_CONTEXT, &reason)))?; + resources_object(py, resources, &held) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(assemble_native_project_resources, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs index efd98b398b..9823ce1309 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs @@ -3,11 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::project_assembly::{ - declared_relations, resources, syntax_checks, syntax_validation, + declared_relations, native_resources, resources, syntax_checks, syntax_validation, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { declared_relations::register(module)?; + native_resources::register(module)?; syntax_validation::register(module)?; syntax_checks::register(module)?; resources::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs index a201a7615d..77eb7ebbfa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs @@ -7,123 +7,16 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyString, PyStringMethods}; use pyo3::{PyErr, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::analysis_session::types::Pairs; -use sqlbuild_analysis::assembly::project::errors::AssemblyError; -use sqlbuild_analysis::assembly::project::main::assemble_project_resources::assemble_project_resources; use sqlbuild_analysis::assembly::project::main::check_sql_syntax::check_sql_syntax; -use sqlbuild_analysis::assembly::project::main::environment_template_names::environment_template_names as template_names; use sqlbuild_analysis::assembly::project::main::sql_syntax_error::sql_syntax_error as check_syntax_error; -use sqlbuild_analysis::assembly::project::models::{ - AuditFacts, FunctionFacts, InputRead, ModelFacts, Namespace, OpaqueRender, OpaqueVariable, - ProjectRequest, ProjectResources, Reference, ResourceFailure, ResourceSlot, SeedDefaults, - SeedFacts, SourceFacts, SyntaxCheck, SyntaxFailure, SyntaxMode, TargetNamespace, Variable, -}; -use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxFailure, SyntaxMode}; use sqlbuild_core::panics::main::native_failure::native_failure; -use sqlbuild_model_config::templates::models::Scalar; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::types::CompilerDetach; -const PROJECT_ASSEMBLY_CONTEXT: &str = "native project assembly"; const SYNTAX_CHECK_CONTEXT: &str = "native SQL syntax check"; -const TEXT_VARIABLE: &str = "text"; -const BOOLEAN_VARIABLE: &str = "bool"; -const NONE_VARIABLE: &str = "none"; -const OPAQUE_VARIABLE: &str = "opaque"; -const RENDERED_TEXT: &str = "text"; -const RENDER_REJECTED: &str = "rejected"; -const RENDER_RAISES: &str = "raises"; -const SOURCE_SLOT: &str = "source"; -const SEED_SLOT: &str = "seed"; -const AUDIT_SLOT: &str = "audit"; -const COMPILE_INPUT_ERROR: &str = "compile_input"; -const VALUE_ERROR: &str = "value"; -const HELD_ERROR: &str = "held"; -const TRUE_TEXT: &str = "1"; -const ENVIRONMENT_READ: &str = "env"; -const CONTEXT_READ: &str = "ctx"; - -/// `(kind, name, package)`. -type ReferenceRow = (String, String, Option); -/// `(database, schema, loader schema)`. -type TargetRow = (Option, Option, Option); -/// `(database, schema, seed database, seed schema)`. -type DefaultsRow = ( - Option, - Option, - Option, - Option, -); -/// `(name, kind, text, render)`: kind `text`, `bool`, `none` or `opaque` with a render pair. -type VariableRow = (String, String, String, Option<(String, String)>); -/// `(resource list, index, error class, message, help, held exception index)`. -type FailureRow = ( - &'static str, - usize, - &'static str, - String, - Option, - Option, -); -/// `(name, directory, tags, references, [(sql, placeholders)])`. -type ModelRow = ( - String, - String, - Vec, - Vec, - Vec<(String, Pairs)>, -); -/// `(name, managed, database, schema)`. -type SourceRow = (String, bool, Option, Option); -/// `(name, tags, database, schema)`. -type SeedRow = (String, Vec, Option, Option); -/// `(kind, name, tags, references)`. -type FunctionRow = (String, String, Vec, Vec); -/// `(references, attached (kind, name))`. -type AuditRow = (Vec, Option<(String, String)>); -/// The dialect, target, defaults, variables, environment and resources one project assembles. -type RequestRow = ( - String, - Option, - DefaultsRow, - Vec, - Vec<(String, Option)>, - Vec, - Vec, - Vec, - Vec, - Vec, -); -/// `(database, schema, logical database, logical schema)`. -type NamespaceRow = ( - Option, - Option, - Option, - Option, -); -/// Deps, managed source and seed namespaces, and the `(env or ctx, name)` template reads. -type ResourcesRow = ( - Vec>, - Vec, Option)>>, - Vec, - Vec>, - Vec>, - Vec<(&'static str, String)>, - Vec, - Option, -); - -/// The project's resource facts up to the resource Python's assembly raises at. -#[pyfunction] -fn assemble_project_resource_facts(py: Python<'_>, request: RequestRow) -> PyResult { - let request: ProjectRequest = project_request(request)?; - let resources: ProjectResources = py - .compiler_detach(|| assemble_project_resources(&request)) - .map_err(|reason| compiler_error(native_failure(PROJECT_ASSEMBLY_CONTEXT, &reason)))?; - resources_row(resources) -} - /// Python's syntax error message for one SQL string, None where it parses. #[pyfunction] #[pyo3(signature = (sql, placeholders, dialect, parse_one=false))] @@ -196,211 +89,8 @@ pub(crate) fn syntax_valid( .map_err(syntax_value_error) } -fn project_request(request: RequestRow) -> PyResult { - let ( - dialect, - target, - defaults, - variables, - environment, - models, - sources, - seeds, - functions, - audits, - ) = request; - let (database, schema, seed_database, seed_schema) = defaults; - Ok(ProjectRequest { - dialect, - target: target.map(|(database, schema, loader_schema)| TargetNamespace { - database, - schema, - loader_schema, - }), - defaults: SeedDefaults { - database, - schema, - seed_database, - seed_schema, - }, - variables: variables - .into_iter() - .map(variable) - .collect::>()?, - environment, - models: models.into_iter().map(model).collect(), - sources: sources - .into_iter() - .map(|(name, managed, database, schema)| SourceFacts { - name, - managed, - database, - schema, - }) - .collect(), - seeds: seeds - .into_iter() - .map(|(name, tags, database, schema)| SeedFacts { - name, - tags, - database, - schema, - }) - .collect(), - functions: functions - .into_iter() - .map(|(kind, name, tags, rows)| FunctionFacts { - kind, - name, - tags, - references: references(rows), - }) - .collect(), - audits: audits - .into_iter() - .map(|(rows, attached)| AuditFacts { - references: references(rows), - attached, - }) - .collect(), - }) -} - -fn variable(row: VariableRow) -> PyResult<(String, Variable)> { - let (name, kind, text, render) = row; - let value: Variable = match (kind.as_str(), render) { - (TEXT_VARIABLE, _) => Variable::Scalar(Scalar::Text(text)), - (BOOLEAN_VARIABLE, _) => Variable::Scalar(Scalar::Bool(text == TRUE_TEXT)), - (NONE_VARIABLE, _) => Variable::Scalar(Scalar::Null), - (OPAQUE_VARIABLE, Some((render_kind, payload))) => Variable::Opaque(OpaqueVariable { - text, - render: opaque_render(&render_kind, payload)?, - }), - _ => { - return Err(compiler_error(native_failure( - PROJECT_ASSEMBLY_CONTEXT, - &format!("an unknown variable kind '{kind}'"), - ))); - } - }; - Ok((name, value)) -} - -fn opaque_render(kind: &str, payload: String) -> PyResult { - match kind { - RENDERED_TEXT => Ok(OpaqueRender::Text(payload)), - RENDER_REJECTED => Ok(OpaqueRender::Rejected(payload)), - RENDER_RAISES => payload.parse().map(OpaqueRender::Raises).map_err(|_| { - compiler_error(native_failure( - PROJECT_ASSEMBLY_CONTEXT, - "an opaque variable's held exception index is not a number", - )) - }), - _ => Err(compiler_error(native_failure( - PROJECT_ASSEMBLY_CONTEXT, - &format!("an unknown variable rendering '{kind}'"), - ))), - } -} - -fn model(row: ModelRow) -> ModelFacts { - let (name, directory, tags, rows, checks) = row; - ModelFacts { - name, - directory, - tags, - references: references(rows), - syntax_checks: checks - .into_iter() - .map(|(sql, placeholders)| SyntaxCheck { sql, placeholders }) - .collect(), - } -} - -fn references(rows: Vec) -> Vec { - rows.into_iter() - .map(|(kind, name, package)| Reference { - kind, - name, - package, - }) - .collect() -} - -fn resources_row(resources: ProjectResources) -> PyResult { - let failure: Option = resources.failure.map(failure_row).transpose()?; - Ok(( - resources.model_deps, - resources.sources, - resources.seeds.into_iter().map(namespace_row).collect(), - resources.function_deps, - resources.audit_deps, - resources.reads.into_iter().map(read_row).collect(), - resources.model_syntax_valid, - failure, - )) -} - -/// `(resource list, index, error class, message, help, held exception index)`. -fn failure_row(failure: ResourceFailure) -> PyResult { - let slot: &'static str = match failure.slot { - ResourceSlot::Source => SOURCE_SLOT, - ResourceSlot::Seed => SEED_SLOT, - ResourceSlot::Audit => AUDIT_SLOT, - }; - Ok(match failure.error { - AssemblyError::CompileInput { message, help } => ( - slot, - failure.index, - COMPILE_INPUT_ERROR, - message, - help, - None, - ), - AssemblyError::Value(message) => (slot, failure.index, VALUE_ERROR, message, None, None), - AssemblyError::Raise(held) => ( - slot, - failure.index, - HELD_ERROR, - String::new(), - None, - Some(held), - ), - AssemblyError::Internal(reason) => { - return Err(compiler_error(native_failure( - PROJECT_ASSEMBLY_CONTEXT, - &reason, - ))); - } - }) -} - -fn read_row(read: InputRead) -> (&'static str, String) { - match read { - InputRead::Environment(name) => (ENVIRONMENT_READ, name), - InputRead::Context(name) => (CONTEXT_READ, name), - } -} - -fn namespace_row(namespace: Namespace) -> NamespaceRow { - ( - namespace.database, - namespace.schema, - namespace.logical_database, - namespace.logical_schema, - ) -} - -/// The distinct `ENV:` names target templates read, in first-seen order. -#[pyfunction] -fn environment_template_names(values: Vec>) -> Vec { - template_names(&values) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(assemble_project_resource_facts, module)?)?; module.add_function(wrap_pyfunction!(check_native_sql_syntax, module)?)?; module.add_function(wrap_pyfunction!(sql_syntax_error, module)?)?; - module.add_function(wrap_pyfunction!(environment_template_names, module)?)?; Ok(()) } diff --git a/python-boundary.toml b/python-boundary.toml index 957d25f2d7..fc561161ec 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -347,7 +347,3 @@ note = "A baseline that does not compile yields no relation keys, as before the [[handled_native_errors]] location = "lint/_helpers/fixes.py::_FixSession._verdict" note = "A candidate fix whose project fails to compile is unverified and not applied, as before the compile entry became native; no Python implementation runs instead." - -[[handled_native_errors]] -location = "compiler/project_assembly/_helpers/request.py::_opaque_render" -note = "A var that cannot render as text is recorded as the error a template using it raises; no Python implementation runs." diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index f81b9a8964..9268966f95 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -188,6 +188,7 @@ from sqlbuild.compiler.planner.models import ( SqlTestPlanEntry, ) from sqlbuild.compiler.planner.types import CursorGrain +from sqlbuild.compiler.project_assembly.models import NativeProjectResources from sqlbuild.compiler.python_nodes.models import ( PythonNodeGraph, PythonNodeIdentity, @@ -2574,7 +2575,6 @@ def select_latest_fingerprints( node_names: tuple[str, ...], filtered_node_types: tuple[str, ...] = (), ) -> FingerprintSet: ... -def environment_template_names(values: list[str | None], /) -> list[str]: ... def find_python_string_locations( *, project_dir: Path, @@ -3320,38 +3320,12 @@ def sql_syntax_error( def check_native_sql_syntax( request: tuple[str, list[tuple[str, list[tuple[str, str]]]]], / ) -> bool: ... -def assemble_project_resource_facts( - request: tuple[ - str, - tuple[str | None, str | None, str | None] | None, - tuple[str | None, str | None, str | None, str | None], - list[tuple[str, str, str, tuple[str, str] | None]], - list[tuple[str, str | None]], - list[ - tuple[ - str, - str, - list[str], - list[tuple[str, str, str | None]], - list[tuple[str, list[tuple[str, str]]]], - ] - ], - list[tuple[str, bool, str | None, str | None]], - list[tuple[str, list[str], str | None, str | None]], - list[tuple[str, str, list[str], list[tuple[str, str, str | None]]]], - list[tuple[list[tuple[str, str, str | None]], tuple[str, str] | None]], - ], - /, -) -> tuple[ - list[list[tuple[str, str]]], - list[tuple[str | None, str | None] | None], - list[tuple[str | None, str | None, str | None, str | None]], - list[list[tuple[str, str]]], - list[list[tuple[str, str]]], - list[tuple[str, str]], - list[bool], - tuple[str, int, str, str, str | None, int | None] | None, -]: ... +def assemble_native_project_resources( + *, + inputs: CompileInputsView, + dialect: str | None, + syntax_checks: tuple[tuple[tuple[str, dict[str, str] | None], ...], ...], +) -> NativeProjectResources: ... def explicit_reference_report( project: Any, settings: dict[str, object], diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py index 5a907dab0f..73283c8cb8 100644 --- a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py +++ b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py @@ -6,6 +6,7 @@ from sqlbuild._native import ( analyze_native_model_sql, + assemble_native_project_resources, infer_native_expression_source_shapes, model_syntax_checks, prove_native_dynamic_contracts, @@ -20,9 +21,6 @@ from sqlbuild.compiler.analysis_session.types import ProofRow from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.project_assembly.main._assemble_native_project_resources import ( - assemble_native_project_resources, -) from sqlbuild.compiler.project_assembly.models import ( NativeModelFacts, NativeProjectFacts, diff --git a/src/sqlbuild/compiler/project_assembly/_helpers/__init__.py b/src/sqlbuild/compiler/project_assembly/_helpers/__init__.py deleted file mode 100644 index 2a1bd2a676..0000000000 --- a/src/sqlbuild/compiler/project_assembly/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native project assembly helpers.""" diff --git a/src/sqlbuild/compiler/project_assembly/_helpers/request.py b/src/sqlbuild/compiler/project_assembly/_helpers/request.py deleted file mode 100644 index 2c890dfe2f..0000000000 --- a/src/sqlbuild/compiler/project_assembly/_helpers/request.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Build the native project assembly request from Python's compile inputs.""" - -from __future__ import annotations - -import os -from collections.abc import Iterable - -import sqlbuild._native as _native -from sqlbuild._native import render_project_var_text -from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.project_assembly.constants import ( - BOOLEAN_VARIABLE, - FALSE_TEXT, - NATIVE_INT_MAX, - NATIVE_INT_MIN, - NONE_VARIABLE, - OPAQUE_VARIABLE, - RENDER_LABEL, - RENDER_RAISES, - RENDER_REJECTED, - RENDERED_TEXT, - TEXT_VARIABLE, - TRUE_TEXT, -) -from sqlbuild.compiler.project_assembly.types import ( - ProjectRequestRow, - ReferenceRow, - SyntaxCheckRow, - VariableRow, -) -from sqlbuild.spec.contracts.models import DefaultsConfig, TargetConfig - - -def project_request( - *, - inputs: CompileInputsView, - dialect: str | None, - syntax_checks: tuple[tuple[tuple[str, dict[str, str] | None], ...], ...], -) -> tuple[ProjectRequestRow, list[Exception | None]]: - """Return the request row and, per variable, the exception rendering it as text raises.""" - - renders: list[tuple[str, str] | Exception | None] = [ - _opaque_render(value=value) for value in inputs.effective_vars.values() - ] - target: TargetConfig | None = inputs.effective_target - defaults: DefaultsConfig = inputs.project_config.defaults - target_values: tuple[str | None, ...] = ( - (target.database, target.schema, target.loader_schema) if target is not None else () - ) - default_values: tuple[str | None, ...] = ( - defaults.database, - defaults.schema, - defaults.seed_database, - defaults.seed_schema, - ) - seed_values: list[str | None] = [] - for seed_input in inputs.seed_inputs: - seed_values.extend((seed_input.schema_entry.database, seed_input.schema_entry.schema)) - row: ProjectRequestRow = ( - dialect or "generic", - (target.database, target.schema, target.loader_schema) if target is not None else None, - (defaults.database, defaults.schema, defaults.seed_database, defaults.seed_schema), - [ - _variable_row(name=name, value=value, render=render, index=index) - for index, ((name, value), render) in enumerate( - zip(inputs.effective_vars.items(), renders, strict=True) - ) - ], - _environment_rows((*target_values, *default_values, *seed_values)), - [ - ( - model_input.model_file.file_path.stem, - str(model_input.model_file.relative_path.parent), - _tag_rows(model_input.config.values.get("tags")), - _reference_rows(model_input.references), - _syntax_check_rows(checks), - ) - for model_input, checks in zip(inputs.model_inputs, syntax_checks, strict=True) - ], - [source_input.assembly_row for source_input in inputs.source_inputs], - [ - ( - seed_input.schema_entry.name, - list(seed_input.schema_entry.tags), - seed_input.schema_entry.database, - seed_input.schema_entry.schema, - ) - for seed_input in inputs.seed_inputs - ], - [function_input.assembly_row for function_input in inputs.sql_function_inputs], - [audit_input.assembly_row for audit_input in inputs.audit_inputs], - ) - return row, [render if isinstance(render, Exception) else None for render in renders] - - -def _tag_rows(value: object) -> list[str]: - """Model config tags as the project graph indexes them.""" - - if isinstance(value, list | tuple): - return [str(item) for item in value] - return [] - - -def _environment_rows(values: Iterable[str | None]) -> list[tuple[str, str | None]]: - """Python's own `os.environ` lookup of every name an `ENV:` template may read.""" - - return [ - (name, os.environ.get(name)) for name in _native.environment_template_names(list(values)) - ] - - -def _variable_row( - *, name: str, value: object, render: tuple[str, str] | Exception | None, index: int -) -> VariableRow: - if value is None: - return (name, NONE_VARIABLE, "", None) - if type(value) is bool: - return (name, BOOLEAN_VARIABLE, TRUE_TEXT if value else FALSE_TEXT, None) - if type(value) is str: - return (name, TEXT_VARIABLE, value, None) - if type(value) is int and NATIVE_INT_MIN <= value <= NATIVE_INT_MAX: - return (name, TEXT_VARIABLE, str(value), None) - held: tuple[str, str] = (RENDER_RAISES, str(index)) - return (name, OPAQUE_VARIABLE, str(value), held if isinstance(render, Exception) else render) - - -def _opaque_render(*, value: object) -> tuple[str, str] | Exception | None: - """`render_project_var_text` for any label: text, the message after the label, or the error.""" - - if value is None or type(value) in (bool, str): - return None - if type(value) is int and NATIVE_INT_MIN <= value <= NATIVE_INT_MAX: - return None - try: - text: str = render_project_var_text(value=value, label=RENDER_LABEL) - except ValueError as error: - message: str = str(error) - if message.startswith(RENDER_LABEL): - return (RENDER_REJECTED, message[len(RENDER_LABEL) :]) - return error - except Exception as error: - return error - return (RENDERED_TEXT, text) - - -def _reference_rows(references: tuple[CompileSqlReference, ...]) -> list[ReferenceRow]: - return [ - (str(reference.ref_kind), reference.ref_name, reference.ref_package) - for reference in references - ] - - -def _syntax_check_rows( - checks: tuple[tuple[str, dict[str, str] | None], ...], -) -> list[SyntaxCheckRow]: - return [(sql, list((placeholders or {}).items())) for sql, placeholders in checks] diff --git a/src/sqlbuild/compiler/project_assembly/_helpers/resources.py b/src/sqlbuild/compiler/project_assembly/_helpers/resources.py deleted file mode 100644 index 7174dd5677..0000000000 --- a/src/sqlbuild/compiler/project_assembly/_helpers/resources.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Materialise the native project assembly result as the objects Python's assembly builds.""" - -from __future__ import annotations - -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.project_assembly.constants import ( - COMPILE_INPUT_FAILURE, - ENVIRONMENT_READ, - HELD_FAILURE, - VALUE_FAILURE, -) -from sqlbuild.compiler.project_assembly.models import NativeProjectResources, NativeResourceFailure -from sqlbuild.compiler.project_assembly.types import ( - FailureRow, - ProjectResourcesRow, -) -from sqlbuild.compiler.sql_analysis.constants import NATIVE_FAILURE_PREFIX - - -def project_resources( - *, inputs: CompileInputsView, row: ProjectResourcesRow, held: list[Exception | None] -) -> NativeProjectResources: - """Return the resource facts the native row describes, in input order, up to the failure.""" - - ( - model_deps, - source_namespaces, - seed_namespaces, - function_deps, - audit_deps, - reads, - syntax, - failure, - ) = row - _ = [_replay_read(kind=kind, name=name) for kind, name in reads] - return NativeProjectResources( - model_deps=tuple(model_deps), - model_syntax_valid=tuple(syntax), - source_namespaces=tuple(source_namespaces), - seed_namespaces=tuple(seed_namespaces), - function_deps=tuple(function_deps), - audit_scope_deps=tuple(audit_deps), - failure=None if failure is None else _failure(row=failure, held=held), - ) - - -def _failure(*, row: FailureRow, held: list[Exception | None]) -> NativeResourceFailure: - slot, index, kind, message, help_text, held_index = row - error: Exception - if kind == COMPILE_INPUT_FAILURE: - error = CompileInputError(message, help=help_text) - elif kind == VALUE_FAILURE: - error = ValueError(message) - elif kind == HELD_FAILURE and held_index is not None and held[held_index] is not None: - error = cast(Exception, held[held_index]) - else: - raise _native.NativeCompilerError( - f"{NATIVE_FAILURE_PREFIX}native project assembly: unknown failure kind '{kind}'" - ) - return NativeResourceFailure(slot=slot, index=index, error=error) - - -def _replay_read(*, kind: str, name: str) -> None: - if kind == ENVIRONMENT_READ: - COMPILE_INPUT_READS.environment_read(name) - else: - COMPILE_INPUT_READS.context_read(name) diff --git a/src/sqlbuild/compiler/project_assembly/constants.py b/src/sqlbuild/compiler/project_assembly/constants.py deleted file mode 100644 index 78c8ec8154..0000000000 --- a/src/sqlbuild/compiler/project_assembly/constants.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Native project assembly request and failure kinds.""" - -TEXT_VARIABLE: str = "text" -BOOLEAN_VARIABLE: str = "bool" -NONE_VARIABLE: str = "none" -OPAQUE_VARIABLE: str = "opaque" -RENDERED_TEXT: str = "text" -RENDER_REJECTED: str = "rejected" -RENDER_RAISES: str = "raises" -RENDER_LABEL: str = "\x00" -COMPILE_INPUT_FAILURE: str = "compile_input" -VALUE_FAILURE: str = "value" -HELD_FAILURE: str = "held" -SOURCE_SLOT: str = "source" -SEED_SLOT: str = "seed" -AUDIT_SLOT: str = "audit" -TRUE_TEXT: str = "1" -FALSE_TEXT: str = "0" -NATIVE_INT_MIN: int = -(2**63) -NATIVE_INT_MAX: int = 2**63 - 1 -ENVIRONMENT_READ: str = "env" diff --git a/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py b/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py deleted file mode 100644 index a16528e4e1..0000000000 --- a/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Assemble the compiled project's resource facts natively.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.project_assembly._helpers.request import project_request -from sqlbuild.compiler.project_assembly._helpers.resources import project_resources -from sqlbuild.compiler.project_assembly.models import NativeProjectResources -from sqlbuild.compiler.project_assembly.types import ProjectRequestRow, ProjectResourcesRow - - -def assemble_native_project_resources( - *, - inputs: CompileInputsView, - dialect: str | None, - syntax_checks: tuple[tuple[tuple[str, dict[str, str] | None], ...], ...], -) -> NativeProjectResources: - """Return the resource facts and the resource Python's assembly raises at, if any.""" - - request: ProjectRequestRow - held: list[Exception | None] - request, held = project_request(inputs=inputs, dialect=dialect, syntax_checks=syntax_checks) - row: ProjectResourcesRow = _native.assemble_project_resource_facts(request) - return project_resources(inputs=inputs, row=row, held=held) diff --git a/src/sqlbuild/compiler/project_assembly/types.py b/src/sqlbuild/compiler/project_assembly/types.py index afcec02958..65e981eb20 100644 --- a/src/sqlbuild/compiler/project_assembly/types.py +++ b/src/sqlbuild/compiler/project_assembly/types.py @@ -2,38 +2,5 @@ from __future__ import annotations -type ReferenceRow = tuple[str, str, str | None] -type SyntaxCheckRow = tuple[str, list[tuple[str, str]]] -type VariableRow = tuple[str, str, str, tuple[str, str] | None] -type FailureRow = tuple[str, int, str, str, str | None, int | None] -type ModelRow = tuple[str, str, list[str], list[ReferenceRow], list[SyntaxCheckRow]] -type SourceRow = tuple[str, bool, str | None, str | None] -type SeedRow = tuple[str, list[str], str | None, str | None] -type FunctionRow = tuple[str, str, list[str], list[ReferenceRow]] -type AuditRow = tuple[list[ReferenceRow], tuple[str, str] | None] -type TargetRow = tuple[str | None, str | None, str | None] -type DefaultsRow = tuple[str | None, str | None, str | None, str | None] -type ProjectRequestRow = tuple[ - str, - TargetRow | None, - DefaultsRow, - list[VariableRow], - list[tuple[str, str | None]], - list[ModelRow], - list[SourceRow], - list[SeedRow], - list[FunctionRow], - list[AuditRow], -] type ObjectKeyRow = tuple[str, str] type NamespaceRow = tuple[str | None, str | None, str | None, str | None] -type ProjectResourcesRow = tuple[ - list[list[ObjectKeyRow]], - list[tuple[str | None, str | None] | None], - list[NamespaceRow], - list[list[ObjectKeyRow]], - list[list[ObjectKeyRow]], - list[tuple[str, str]], - list[bool], - FailureRow | None, -] diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py index cd500ba70c..17fd7216d5 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py @@ -52,17 +52,6 @@ class OptOutTestCase: expected_rejected: dict[str, bool] -@dataclass(frozen=True) -class GraphFactsTestCase: - """A project whose assembly request carries each resource's project-graph facts.""" - - description: str - files: dict[str, str] - expected_model_tags: tuple[list[str], ...] - expected_seeds: list[tuple[str, list[str]]] - expected_function_kinds: tuple[str, ...] - - @dataclass(frozen=True) class NativePanicTestCase: """A project whose native assembly panics in the parser.""" diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py index 2b3f192183..4c3a914582 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -25,7 +25,6 @@ CompileAdapterContext, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.project_assembly.types import ProjectRequestRow from sqlbuild.sql_values.types import CollectionRendering _PYTHON_RESOURCE_FUNCTIONS: tuple[str, ...] = ("validate_sql_syntax", "validate_hook_sql_syntax") @@ -244,20 +243,6 @@ def python_resource_calls(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: return calls -def record_assembly_requests(*, monkeypatch: pytest.MonkeyPatch) -> list[ProjectRequestRow]: - """Record every request the native project assembly receives.""" - - requests: list[ProjectRequestRow] = [] - original: Callable[..., Any] = _native.assemble_project_resource_facts - - def recording(request: ProjectRequestRow) -> Any: - requests.append(request) - return original(request) - - monkeypatch.setattr(_native, "assemble_project_resource_facts", recording) - return requests - - def _write(*, path: Path, contents: str) -> int: path.parent.mkdir(parents=True, exist_ok=True) return path.write_text(contents, encoding="utf-8") diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py index 795b552976..652a09570b 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py @@ -14,7 +14,6 @@ from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.project_assembly.types import ProjectRequestRow from tests.integration.src.sqlbuild.compiler.helpers import ( GoldenEntry, golden_differences, @@ -26,7 +25,6 @@ AssemblyDeferralTestCase, EdgeAssemblyTestCase, GeneratedAssemblyParityTestCase, - GraphFactsTestCase, NativePanicTestCase, OptOutTestCase, WindowsEnvironmentTestCase, @@ -38,7 +36,6 @@ generated_assembly_files, project_inputs, python_resource_calls, - record_assembly_requests, recorded_assembly, rejected_opt_outs, seed_yml, @@ -251,68 +248,6 @@ def test_given_sql_analysis_opt_outs_when_attaching_natively_then_python_rejecti assert native == test_case.expected_rejected -@pytest.mark.parametrize( - "test_case", - [ - GraphFactsTestCase( - description="tagged models in folders, tagged seeds and both function kinds", - files={ - "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', - "models/staging/orders.sql": ( - "MODEL (description 'Orders.', tags [daily, finance]);\n\nSELECT 1 AS id\n" - ), - "models/customers.sql": ( - "MODEL (description 'Customers.');\n\nSELECT id FROM __ref(\"orders\")\n" - ), - "seeds/regions.yml": ( - "seeds:\n - name: regions\n description: Regions.\n" - " tags: [lookup]\n columns:\n - name: n\n type: INTEGER\n" - ), - "seeds/regions.csv": "n\n1\n", - "functions/sql/is_big.sql": ( - "FUNCTION (description 'Big.', arguments (x INTEGER), returns BOOLEAN," - " tags [math]);\n\nx > 100\n" - ), - "functions/sql/order_ids.sql": ( - "FUNCTION (description 'Ids.', arguments (x INTEGER)," - ' returns table (id INTEGER));\n\nSELECT id FROM __ref("orders")\n' - ), - }, - expected_model_tags=([], ["daily", "finance"]), - expected_seeds=[("regions", ["lookup"])], - expected_function_kinds=("udf", "table_fn"), - ) - ], - ids=lambda case: case.description, -) -def test_given_project_when_assembling_natively_then_request_carries_graph_facts( - test_case: GraphFactsTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - requests: list[ProjectRequestRow] = record_assembly_requests(monkeypatch=monkeypatch) - project: CompiledProjectView = assemble_with( - inputs=project_inputs(project_dir=tmp_path / "project", files=test_case.files), - monkeypatch=monkeypatch, - ) - request: ProjectRequestRow = requests[0] - - assert ( - [(name, directory, tags) for name, directory, tags, _, _ in request[5]], - [(name, tags) for name, tags, _, _ in request[7]], - [(kind, name, tags) for kind, name, tags, _ in request[8]], - ) == ( - [ - (model.key.name, str(model.relative_path.parent)) + (tags,) - for model, tags in zip(project.models, test_case.expected_model_tags, strict=True) - ], - test_case.expected_seeds, - [ - (str(function.key.resource_type), function.key.name, list(function.tags)) - for function in project.functions - ], - ) - assert [kind for kind, _, _, _ in request[8]] == list(test_case.expected_function_kinds) - - @pytest.mark.parametrize( "test_case", [ From 996a17216528bd4bf3cbc0698d0fd035114ba7b4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 03:44:29 +0100 Subject: [PATCH 10/40] test: derive audit scope dependencies through the native assembly binding --- .../compiler/planner/_helpers/helpers.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 61383e156a..4f60c4a0a5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -8,8 +8,8 @@ from datetime import UTC, datetime from itertools import chain from pathlib import Path -from types import MappingProxyType -from typing import ClassVar +from types import MappingProxyType, SimpleNamespace +from typing import Any, ClassVar import sqlbuild._native as _native from sqlbuild._native import ( @@ -83,6 +83,7 @@ from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import ( + DefaultsConfig, SchemaColumn, SchemaModelEntry, SchemaSeedEntry, @@ -1478,9 +1479,19 @@ def native_audit_scope_deps( ], attached, ) - audit_deps: list[list[tuple[str, str]]] = _native.assemble_project_resource_facts( - ("generic", None, (None, None, None, None), [], [], [], [], [], [], [audit_row]) - )[4] + inputs: Any = SimpleNamespace( + effective_vars={}, + effective_target=None, + project_config=SimpleNamespace(defaults=DefaultsConfig()), + seed_inputs=(), + model_inputs=(), + source_inputs=(), + sql_function_inputs=(), + audit_inputs=(SimpleNamespace(assembly_row=audit_row),), + ) + audit_deps: tuple[list[tuple[str, str]], ...] = _native.assemble_native_project_resources( + inputs=inputs, dialect=None, syntax_checks=() + ).audit_scope_deps return tuple( CompiledObjectKey(resource_type=CompiledResourceType(kind), name=name) for kind, name in audit_deps[0] From 75d21f565045fe7e4a0bbfd52aff51c7ea6ef2c4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 04:03:44 +0100 Subject: [PATCH 11/40] style: name the native assembly request variable rows --- .../_helpers/project_assembly/native_resources.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs index bc35313996..87c54458bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs @@ -44,6 +44,9 @@ type FunctionRow = (String, String, Vec, Vec); /// `(references, attached (kind, name))`. type AuditRow = (Vec, Option<(String, String)>); +/// The request's named variables and the Python exceptions opaque ones raise, by index. +type RequestVariables<'py> = (Vec<(String, Variable)>, Vec>); + /// How one opaque variable renders as text, or the Python exception rendering it raises. enum Render<'py> { Text(String), @@ -91,9 +94,7 @@ fn opaque_render<'py>(value: &Bound<'py, PyAny>) -> PyResult> { } /// Each effective var as the native request reads it, and the exceptions opaque ones raise. -fn variables<'py>( - effective_vars: &Bound<'py, PyAny>, -) -> PyResult<(Vec<(String, Variable)>, Vec>)> { +fn variables<'py>(effective_vars: &Bound<'py, PyAny>) -> PyResult> { let mut rows: Vec<(String, Variable)> = Vec::new(); let mut held: Vec> = Vec::new(); for item in effective_vars.call_method0("items")?.try_iter()? { @@ -152,7 +153,7 @@ fn tags(value: &Bound<'_, PyAny>) -> PyResult> { } value .try_iter()? - .map(|item| Ok(item?.str()?.extract::()?)) + .map(|item| item?.str()?.extract::()) .collect() } From 620e6c82f84334a5d66a179ef8f9c53a62135bdc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 04:06:26 +0100 Subject: [PATCH 12/40] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index fc561161ec..7451ff087d 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 59985 +remaining_lines = 58914 [[stays]] path = "executor" From 3352221c4e41e59325129186335747efbf363e26 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 04:21:10 +0100 Subject: [PATCH 13/40] refactor: resolve spec configuration natively Target name and config, effective adapter name, collection rendering, scenario config and dialect overrides, loader destination parts and dynamic family matching are native bindings. The spec/contracts main and _helpers wrappers and the NamePattern class go; Rust callers use sqlbuild._native. --- .../_helpers/build_command/invocation.rs | 6 +- .../bindings/_helpers/clone/clone_command.rs | 6 +- .../_helpers/clone/clone_execution.rs | 2 +- .../_helpers/command_runtime/connection.rs | 6 +- .../_helpers/command_runtime/warehouses.rs | 4 +- .../_helpers/compile_attachment/target.rs | 4 +- .../_helpers/compile_driver/analysis.rs | 2 +- .../compile_driver/assembly_project.rs | 6 +- .../compile_driver/config_validation.rs | 4 +- .../_helpers/compile_driver/inputs.rs | 2 +- .../_helpers/compile_driver/phases.rs | 2 +- .../_helpers/compile_driver/project.rs | 5 +- .../_helpers/compile_driver/targets.rs | 4 +- .../_helpers/contracts/dynamic_names.rs | 32 -- .../_helpers/contracts/effective_config.rs | 4 +- .../src/bindings/_helpers/contracts/mod.rs | 2 +- .../_helpers/contracts/registration.rs | 4 +- .../_helpers/contracts/spec_resolution.rs | 343 ++++++++++++++++++ .../bindings/_helpers/cost_command/refresh.rs | 2 +- .../_helpers/debug_command/run_debug.rs | 2 +- .../bindings/_helpers/diff_report/command.rs | 4 +- .../bindings/_helpers/diff_report/query.rs | 2 +- .../_helpers/format_command/selection.rs | 2 +- .../_helpers/freshness/freshness_command.rs | 2 +- .../_helpers/janitor/janitor_command.rs | 2 +- .../_helpers/lineage/lineage_command.rs | 2 +- .../bindings/_helpers/load/load_command.rs | 2 +- .../bindings/_helpers/load/load_selection.rs | 4 +- .../_helpers/query_command/project.rs | 2 +- .../_helpers/refactor/contract_compare.rs | 2 +- .../_helpers/scenario_command/setup.rs | 5 +- .../bindings/_helpers/scopes/scope_loading.rs | 5 +- fensu.toml | 5 + python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 30 +- .../_helpers/build_planning/defer_clone.py | 7 +- .../cli/commands/_helpers/rules/command.py | 9 +- .../_helpers/runtime/adapter_context.py | 5 +- .../compile/main/_effective_target.py | 3 +- .../compiler/compile/main/effective_config.py | 4 +- .../compile/main/effective_runtime.py | 4 +- .../compile/main/sql_expansion_context.py | 8 +- .../compiler/pipeline/_helpers/clone.py | 3 +- .../planner/_helpers/changes/schema.py | 2 +- .../planner/_helpers/graph/loader_dag.py | 2 +- .../planner/_helpers/planning/retention.py | 2 +- .../_helpers/warehouse/source_deferral.py | 3 +- .../executor/pipeline/_helpers/scenario.py | 4 +- .../run/_helpers/validation/contracts.py | 2 +- .../dbt/_helpers/pipeline/execution_phases.py | 2 +- .../dbt/_helpers/pipeline/interop_prologue.py | 4 +- src/sqlbuild/lint/_helpers/expansion.py | 4 +- .../spec/contracts/_helpers/__init__.py | 0 .../contracts/_helpers/dynamic_columns.py | 33 -- .../spec/contracts/_helpers/project_config.py | 93 ----- .../spec/contracts/_helpers/targets.py | 61 ---- src/sqlbuild/spec/contracts/constants.py | 2 - src/sqlbuild/spec/contracts/main/__init__.py | 0 .../main/loader_destination_parts.py | 30 -- .../main/matching_dynamic_families.py | 12 - .../main/resolve_effective_adapter_name.py | 17 - .../resolve_effective_collection_rendering.py | 20 - .../main/resolve_effective_scenario_config.py | 17 - .../contracts/main/resolve_target_config.py | 21 -- .../contracts/main/resolve_target_name.py | 19 - ...enario_local_type_overrides_for_dialect.py | 17 - .../src/sqlbuild/compiler/compile/helpers.py | 4 +- .../sqlbuild/compiler/model_config/helpers.py | 10 +- .../commands/_helpers/cost/test_refresh.py | 6 +- .../commands/main/compile/test_adapters.py | 4 +- .../cli/commands/main/entry/helpers.py | 3 +- .../sqlbuild/compiler/compile/test_main.py | 3 +- .../discovery/_helpers/test_target_diff.py | 2 +- .../spec/contracts/main/test_constants.py | 4 +- .../main/test_loader_destination_parts.py | 2 +- .../test_resolve_effective_scenario_config.py | 5 +- .../spec/contracts/main/test_targets.py | 2 +- 77 files changed, 468 insertions(+), 500 deletions(-) delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/contracts/dynamic_names.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs delete mode 100644 src/sqlbuild/spec/contracts/_helpers/__init__.py delete mode 100644 src/sqlbuild/spec/contracts/_helpers/dynamic_columns.py delete mode 100644 src/sqlbuild/spec/contracts/_helpers/project_config.py delete mode 100644 src/sqlbuild/spec/contracts/_helpers/targets.py delete mode 100644 src/sqlbuild/spec/contracts/main/__init__.py delete mode 100644 src/sqlbuild/spec/contracts/main/loader_destination_parts.py delete mode 100644 src/sqlbuild/spec/contracts/main/matching_dynamic_families.py delete mode 100644 src/sqlbuild/spec/contracts/main/resolve_effective_adapter_name.py delete mode 100644 src/sqlbuild/spec/contracts/main/resolve_effective_collection_rendering.py delete mode 100644 src/sqlbuild/spec/contracts/main/resolve_effective_scenario_config.py delete mode 100644 src/sqlbuild/spec/contracts/main/resolve_target_config.py delete mode 100644 src/sqlbuild/spec/contracts/main/resolve_target_name.py delete mode 100644 src/sqlbuild/spec/contracts/main/scenario_local_type_overrides_for_dialect.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 7ae951885e..94492d1078 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -17,9 +17,9 @@ const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const REPORTERS_MODULE: &str = "sqlbuild.cli.progress.main._build_command_progress_reporters"; const SETTINGS_MODULE: &str = "sqlbuild.compiler.compile.main.effective_settings"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; -const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; -const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; +const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; const DEFER_CONFLICT_CODE: &str = "C408"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index 50470d396a..32d0ebd4ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -28,9 +28,9 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; -const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const TARGET_NAME_MODULE: &str = "sqlbuild._native"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const CLONE_PIPELINE_MODULE: &str = "sqlbuild.compiler.pipeline.main._clone"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs index 742ff72c25..c5f6e01005 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs @@ -23,7 +23,7 @@ const TRANSIENT_MODELS_MODULE: &str = "sqlbuild.executor.clone.main.build_destination_transient_models"; const FINGERPRINTING_MODULE: &str = "sqlbuild.executor.clone.main.fingerprinting"; const EXECUTE_MODULE: &str = "sqlbuild.executor.clone.main.execute"; -const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const DESTINATION_ENTRY_GROUPS: [&str; 4] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 7ee3a0cc7d..2b888b8f70 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -21,9 +21,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const DBT_PROFILE_MODULE: &str = "sqlbuild.integrations.dbt.main.profile.profile_connection"; const DBT_ERRORS_MODULE: &str = "sqlbuild.integrations.dbt.exceptions"; const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; -const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; -const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; +const TARGET_NAME_MODULE: &str = "sqlbuild._native"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const CONNECTION_OPTIONS: (&[&str], &[&str]) = ( &["raw_config", "project_dir", "adapter_name"], &["discovered_inputs", "cli_vars", "selected_target"], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index 6677cb91ee..195988a36a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -26,8 +26,8 @@ use crate::bindings::_helpers::discovery::configuration::discover_project_config const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; const SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; -const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const TARGET_NAME_MODULE: &str = "sqlbuild._native"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main._session_warehouse_support"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; const EXPAND_TEMPLATE_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs index 1904f73910..23fc340a8d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -10,8 +10,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::model_config::template_data::{early_model_templates, run_id}; -const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; -const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const TARGET_NAME_MODULE: &str = "sqlbuild._native"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const FRONTIER_CONSTANTS_MODULE: &str = "sqlbuild.compiler.frontier.constants"; const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index ff64912957..31a965b2b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const DISCOVER_MODULE: &str = "sqlbuild._native"; const REJECT_SELECTORS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const SELECTED_GRAPH_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index c4cf4b2e98..ec227ec603 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -21,11 +21,9 @@ use crate::bindings::_helpers::scopes::scope_errors::report_scope_index_errors; const NATIVE_MODULE: &str = "sqlbuild._native"; const SCOPE_USAGES_MODULE: &str = "sqlbuild.compiler.compile.main._scope_index_with_compile_usages"; -pub(crate) const ADAPTER_NAME_MODULE: &str = - "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +pub(crate) const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTER_TYPES_MODULE: &str = "sqlbuild.adapter.contract.types"; -const SCENARIO_CONFIG_MODULE: &str = - "sqlbuild.spec.contracts.main.resolve_effective_scenario_config"; +const SCENARIO_CONFIG_MODULE: &str = "sqlbuild._native"; const SQL_TESTS_MODULE: &str = "sqlbuild.compiler.compile._helpers.native_stages.sql_tests"; const RECOVERY_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.recovery"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs index fee36f43b6..68fcd2cedf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs @@ -9,8 +9,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; const SPEC_ERRORS_MODULE: &str = "sqlbuild.spec.contracts.exceptions"; -const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main._session_warehouse_support"; const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index 201a79d8a2..d4ad6d5525 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -19,7 +19,7 @@ const ATTACH_CHECKS_MODULE: &str = "sqlbuild._native"; const ATTACH_RESOURCES_MODULE: &str = "sqlbuild._native"; const INPUTS_VIEW_MODULE: &str = "sqlbuild._native"; const INPUTS_WITH_MODULE: &str = "sqlbuild._native"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const REQUIRED: [&str; 2] = ["discovered_inputs", "adapter_context"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index ce8694d646..6cec2848d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{Status, call_with, use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; const MANIFEST_MODULE: &str = "sqlbuild.compiler.manifest.main._build_graph_manifest"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const PYTHON_NODES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild.cli.output.models"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 69219b03c4..2f8728e766 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -28,9 +28,8 @@ const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants" const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.analysis_selection"; const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._record"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; -const COLLECTION_RENDERING_MODULE: &str = - "sqlbuild.spec.contracts.main.resolve_effective_collection_rendering"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; +const COLLECTION_RENDERING_MODULE: &str = "sqlbuild._native"; /// Optional keywords of the project graph builders. const GRAPH_OPTIONAL: [&str; 7] = [ "selected_target", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index 1d58698350..4d04157e08 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -23,8 +23,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; -const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; -const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const TARGET_NAME_MODULE: &str = "sqlbuild._native"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; const EXPAND_TEMPLATE_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/dynamic_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/dynamic_names.rs deleted file mode 100644 index 3f6575ad8f..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/dynamic_names.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! Compiled dynamic column family `name_pattern`s for matching runtime column names. - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyclass, pymethods}; -use sqlbuild_model_config::name_patterns::main::compile_name_pattern::compile_name_pattern; -use sqlbuild_model_config::name_patterns::models::NamePattern; - -/// A `name_pattern` compiled to match whole column names; an invalid pattern raises `ValueError`. -#[pyclass(name = "NamePattern", module = "sqlbuild._native", frozen)] -struct PyNamePattern { - pattern: NamePattern, -} - -#[pymethods] -impl PyNamePattern { - #[new] - fn new(pattern: &str) -> PyResult { - compile_name_pattern(pattern) - .map(|pattern| Self { pattern }) - .map_err(|error| PyValueError::new_err(format!("invalid name_pattern: {error}"))) - } - - /// Whether `column_name` wholly matches the pattern. - fn matches(&self, column_name: &str) -> bool { - self.pattern.matches(column_name) - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs index 8ec58220c6..18679760d8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs @@ -180,7 +180,7 @@ fn message<'py>( /// `resolve_target_name`: the effective target, `None`, or the unknown-target message. #[pyfunction] -fn effective_config_target_name<'py>( +pub(crate) fn effective_config_target_name<'py>( py: Python<'py>, project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, @@ -202,7 +202,7 @@ fn effective_config_target_name<'py>( /// `resolve_target_config`: each merged `TargetConfig` field, or the ambiguous-connection message. #[pyfunction] -fn effective_config_target_fields<'py>( +pub(crate) fn effective_config_target_fields<'py>( py: Python<'py>, configs: (Bound<'py, PyAny>, Bound<'py, PyAny>), target_name: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs index 38d21f7a13..4eadc39c96 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs @@ -1,8 +1,8 @@ pub(crate) mod contract_diagnostics; -pub(crate) mod dynamic_names; pub(crate) mod effective_config; pub(crate) mod model_contracts; pub(crate) mod project_config; pub(crate) mod registration; pub(crate) mod resource_identities; +pub(crate) mod spec_resolution; pub(crate) mod sql_values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs index 846c94987a..6d04c9a5cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs @@ -3,15 +3,15 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::contracts::{ - contract_diagnostics, dynamic_names, effective_config, project_config, resource_identities, + contract_diagnostics, effective_config, project_config, resource_identities, spec_resolution, sql_values, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { contract_diagnostics::register(module)?; - dynamic_names::register(module)?; effective_config::register(module)?; project_config::register(module)?; resource_identities::register(module)?; + spec_resolution::register(module)?; sql_values::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs new file mode 100644 index 0000000000..097936c985 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs @@ -0,0 +1,343 @@ +//! Effective target, adapter, rendering, scenario and loader destination resolution. + +use std::cell::RefCell; +use std::collections::HashMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_model_config::name_patterns::main::compile_name_pattern::compile_name_pattern; +use sqlbuild_model_config::name_patterns::models::NamePattern; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::contracts::effective_config::{ + effective_config_target_fields, effective_config_target_name, +}; + +const MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const ERRORS_MODULE: &str = "sqlbuild.spec.contracts.exceptions"; +const TARGET_GROUPS: [(&str, &str); 4] = [ + ("clone", "ClonePolicy"), + ("execution_limits", "ExecutionLimitsConfig"), + ("diff", "TargetDiffConfig"), + ("warehouses", "TargetWarehousesConfig"), +]; +const SNAPSHOT_LIMITS: [&str; 4] = [ + "max_rows_per_relation", + "max_total_rows", + "max_bytes_per_relation", + "max_total_bytes", +]; +const ALL_DIALECTS: &str = "*"; +const LOADER_SCHEMA_TABLE_PARTS: usize = 2; +const LOADER_QUALIFIED_TABLE_PARTS: usize = 3; +const NAME_PATTERN_CACHE_LIMIT: usize = 256; + +thread_local! { + static NAME_PATTERNS: RefCell> = RefCell::new(HashMap::new()); +} + +fn spec_error<'py>(py: Python<'py>, message: Bound<'py, PyString>) -> PyResult { + Ok(PyErr::from_value( + imported(py, ERRORS_MODULE, "SpecConfigError")?.call1((message,))?, + )) +} + +/// Resolve the effective target name. +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config, selected_target))] +pub(crate) fn resolve_target_name<'py>( + py: Python<'py>, + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, + selected_target: &Bound<'py, PyAny>, +) -> PyResult>> { + let (name, error) = + effective_config_target_name(py, project_config, local_config, selected_target)?; + match error { + Some(message) => Err(spec_error(py, message)?), + None => Ok(name), + } +} + +/// Merge project target config with local developer overrides. +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config, target_name))] +pub(crate) fn resolve_target_config<'py>( + py: Python<'py>, + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, + target_name: &Bound<'py, PyAny>, +) -> PyResult> { + let target_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "TargetConfig")?; + let (rows, error) = effective_config_target_fields( + py, + (project_config.clone(), local_config.clone()), + target_name, + &target_class.call0()?, + )?; + let Some(rows) = rows else { + let message: Bound<'py, PyString> = error.unwrap_or_else(|| PyString::new(py, "")); + return Err(spec_error(py, message)?); + }; + let values: Bound<'py, PyDict> = PyDict::new(py); + let grouped: Vec<(&str, Bound<'py, PyDict>)> = TARGET_GROUPS + .iter() + .map(|(group, _)| (*group, PyDict::new(py))) + .collect(); + for (group, name, value) in rows { + match group { + None => values.set_item(name, value)?, + Some(group) => { + if let Some((_, fields)) = grouped.iter().find(|(known, _)| *known == group) { + fields.set_item(name, value)?; + } + } + } + } + for ((group, class_name), (_, fields)) in TARGET_GROUPS.iter().zip(&grouped) { + let nested: Bound<'py, PyAny> = + imported(py, MODELS_MODULE, class_name)?.call((), Some(fields))?; + values.set_item(*group, nested)?; + } + target_class.call((), Some(&values)) +} + +/// Resolve the effective adapter name, allowing local override. +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config))] +pub(crate) fn resolve_effective_adapter_name<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, +) -> PyResult> { + let local: Bound<'py, PyAny> = local_config.getattr("adapter")?; + if local.is_none() { + return project_config.getattr("adapter"); + } + Ok(local) +} + +/// Resolve declaration rendering over project configuration. +#[pyfunction] +#[pyo3(signature = (*, project_config, declaration_override))] +pub(crate) fn resolve_effective_collection_rendering<'py>( + project_config: &Bound<'py, PyAny>, + declaration_override: &Bound<'py, PyAny>, +) -> PyResult> { + if !declaration_override.is_none() { + return Ok(declaration_override.clone()); + } + project_config + .getattr("constants")? + .getattr("collection_rendering") +} + +/// `local` when set, else `project`. +fn override_or<'py>(local: Bound<'py, PyAny>, project: Bound<'py, PyAny>) -> Bound<'py, PyAny> { + if local.is_none() { project } else { local } +} + +/// Resolve scenario config, allowing local overrides to replace project rules. +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config))] +fn resolve_effective_scenario_config<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_config.py(); + let project: Bound<'py, PyAny> = project_config.getattr("scenario")?; + let local: Bound<'py, PyAny> = local_config.getattr("scenario")?; + let overrides: Bound<'py, PyDict> = PyDict::new(py); + for item in project + .getattr("local_type_overrides")? + .call_method0("items")? + .try_iter()? + { + let (dialect, rules): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + overrides.set_item(dialect, py.get_type::().call1((rules,))?)?; + } + for item in local + .getattr("local_type_overrides")? + .call_method0("items")? + .try_iter()? + { + let (dialect, rules): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + overrides + .call_method1("setdefault", (dialect, PyDict::new(py)))? + .call_method1("update", (rules,))?; + } + let project_limits: Bound<'py, PyAny> = project.getattr("snapshot_limits")?; + let local_limits: Bound<'py, PyAny> = local.getattr("snapshot_limits")?; + let limits: Vec<(&str, Bound<'py, PyAny>)> = SNAPSHOT_LIMITS + .iter() + .map(|field| { + Ok(( + *field, + override_or( + local_limits.getattr(*field)?, + project_limits.getattr(*field)?, + ), + )) + }) + .collect::>()?; + call_with( + &imported(py, MODELS_MODULE, "ScenarioConfig")?, + &[ + ( + "run_namespace", + override_or( + local.getattr("run_namespace")?, + project.getattr("run_namespace")?, + ), + ), + ("local_type_overrides", overrides.into_any()), + ( + "snapshot_limits", + call_with( + &imported(py, MODELS_MODULE, "ScenarioSnapshotLimitsConfig")?, + &limits, + )?, + ), + ], + ) +} + +/// Return global and dialect-specific scenario local type override rules. +#[pyfunction] +#[pyo3(signature = (*, scenario_config, sql_analysis_dialect))] +fn scenario_local_type_overrides_for_dialect<'py>( + scenario_config: &Bound<'py, PyAny>, + sql_analysis_dialect: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = scenario_config.py(); + let rules: Bound<'py, PyAny> = scenario_config.getattr("local_type_overrides")?; + let overrides: Bound<'py, PyDict> = PyDict::new(py); + overrides.call_method1( + "update", + (rules.call_method1("get", (ALL_DIALECTS, PyDict::new(py)))?,), + )?; + if !sql_analysis_dialect.is_none() { + overrides.call_method1( + "update", + (rules.call_method1("get", (sql_analysis_dialect, PyDict::new(py)))?,), + )?; + } + Ok(overrides) +} + +/// Whether `text` is empty or only whitespace as Python's `str.strip` sees it. +fn blank(text: &str) -> bool { + text.chars() + .all(|character| character.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&character)) +} + +/// Parse a one-, two-, or three-part loader destination. +#[pyfunction] +#[pyo3(signature = (*, destination, default_database, default_schema))] +fn loader_destination_parts<'py>( + py: Python<'py>, + destination: &str, + default_database: Option>, + default_schema: Option>, +) -> PyResult> { + loader_parts(py, destination, (default_database, default_schema)) +} + +fn loader_parts<'py>( + py: Python<'py>, + destination: &str, + (default_database, default_schema): (Option>, Option>), +) -> PyResult> { + let parts: Vec<&str> = destination.split('.').collect(); + if !(1..=LOADER_QUALIFIED_TABLE_PARTS).contains(&parts.len()) + || parts.iter().any(|part| blank(part)) + { + return Err(spec_error( + py, + PyString::new( + py, + &format!( + "Invalid loader destination '{destination}': expected 1 to 3 non-empty \ + dot-separated parts (table, schema.table, or database.schema.table)" + ), + ), + )?); + } + let none = || py.None().into_bound(py); + let text = |part: &str| PyString::new(py, part).into_any(); + let (database, schema, table): (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>) = + match parts.as_slice() { + [table] => ( + default_database.unwrap_or_else(none), + default_schema.unwrap_or_else(none), + text(table), + ), + [schema, table] if parts.len() == LOADER_SCHEMA_TABLE_PARTS => ( + default_database.unwrap_or_else(none), + text(schema), + text(table), + ), + _ => (text(parts[0]), text(parts[1]), text(parts[2])), + }; + imported(py, MODELS_MODULE, "LoaderDestinationParts")?.call1((database, schema, table)) +} + +/// Whether a family's optional `name_pattern` accepts `column_name`. +fn family_accepts(family: &Bound<'_, PyAny>, column_name: &str) -> PyResult { + let pattern: Option = family.getattr("name_pattern")?.extract()?; + let Some(pattern) = pattern else { + return Ok(true); + }; + NAME_PATTERNS.with(|cache| { + let mut cache = cache.borrow_mut(); + if !cache.contains_key(&pattern) { + let compiled: NamePattern = compile_name_pattern(&pattern) + .map_err(|error| PyValueError::new_err(format!("invalid name_pattern: {error}")))?; + if cache.len() >= NAME_PATTERN_CACHE_LIMIT { + cache.clear(); + } + cache.insert(pattern.clone(), compiled); + } + Ok(cache + .get(&pattern) + .is_some_and(|compiled| compiled.matches(column_name))) + }) +} + +/// Return declared dynamic families accepting a physical column name. +#[pyfunction] +#[pyo3(signature = (*, families, column_name))] +fn matching_dynamic_families<'py>( + families: &Bound<'py, PyAny>, + column_name: &str, +) -> PyResult> { + let mut matching: Vec> = Vec::new(); + for family in families.try_iter()? { + let family: Bound<'py, PyAny> = family?; + if family_accepts(&family, column_name)? { + matching.push(family); + } + } + PyTuple::new(families.py(), matching) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_target_name, module)?)?; + module.add_function(wrap_pyfunction!(resolve_target_config, module)?)?; + module.add_function(wrap_pyfunction!(resolve_effective_adapter_name, module)?)?; + module.add_function(wrap_pyfunction!( + resolve_effective_collection_rendering, + module + )?)?; + module.add_function(wrap_pyfunction!(resolve_effective_scenario_config, module)?)?; + module.add_function(wrap_pyfunction!( + scenario_local_type_overrides_for_dialect, + module + )?)?; + module.add_function(wrap_pyfunction!(loader_destination_parts, module)?)?; + module.add_function(wrap_pyfunction!(matching_dynamic_families, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs index f979330cd9..c7ae5a8a37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild.compiler.compile.main._effective_target"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const COST_TYPES: &str = "sqlbuild.cost.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 84ad9cafc5..069ca675fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -35,7 +35,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const DEBUG_OPTIONS: (&[&str], &[&str]) = ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs index bd17fa0151..7b92d284e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -31,8 +31,8 @@ use crate::bindings::_helpers::frontier::encoder::emit_error; const NATIVE_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; -const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const TARGET_NAME_MODULE: &str = "sqlbuild._native"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const CONTRACT_CONSTANTS_MODULE: &str = "sqlbuild.spec.contracts.constants"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const EXECUTE_MODULE: &str = "sqlbuild.executor.diff.main.execute"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index 6efc6275fc..7a9c977cd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -18,7 +18,7 @@ use crate::bindings::_helpers::diff_report::support::{ }; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const NAMESPACE_MODULE: &str = "sqlbuild.compiler.compile.main._effective_target_namespace"; const RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index 1508e8bbdf..01508c0bdf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -23,7 +23,7 @@ const UNIT_TEST_SELECTORS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const SELECTION_CODE: &str = "S007"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index 2bc4885d43..a6f4a96e63 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -22,7 +22,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const CONNECTION_MODULE: &str = "sqlbuild._native"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index eb83130059..6d3bc6bb69 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -26,7 +26,7 @@ const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const PREVIEW_TARGET_MODULE: &str = "sqlbuild._native"; -const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; +const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const RETENTION_MODULE: &str = "sqlbuild.executor.janitor.main.resolve_retention_days"; const PLANNING_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.planning"; const EXECUTION_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.execution"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index a1fdf1b70f..0296e7af8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -41,7 +41,7 @@ const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const STATUS_MODULE: &str = "sqlbuild.presentation.main._maybe_status"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; const SELECTION_MODULE: &str = "sqlbuild.compiler.planner.main.selection.selection"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 931f832f8e..97791c63fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -31,7 +31,7 @@ const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_u const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild.compiler.compile.main._effective_target"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild.compiler.compile.main.effective_settings"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; const CURSOR_OVERRIDES_MODULE: &str = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 62c40b9cbe..af5cb07dd4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -18,9 +18,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; -const DESTINATION_MODULE: &str = "sqlbuild.spec.contracts.main.loader_destination_parts"; +const DESTINATION_MODULE: &str = "sqlbuild._native"; const CONNECTION_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTER_TYPES_MODULE: &str = "sqlbuild.adapter.contract.types"; const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; const LOADER_TABLE_PREFIX: &str = "__loader__"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs index 5341d58eed..01c92d2a40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs @@ -5,7 +5,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; /// The project directory, defaulting to the working directory, with its inputs and adapter. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs index e180f53fa8..1ac9d93180 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs @@ -13,7 +13,7 @@ const ADOPTION_TYPES_MODULE: &str = "sqlbuild.compiler.contract_adoption.types"; const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const TYPES_EQUAL_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; -const DYNAMIC_FAMILIES_MODULE: &str = "sqlbuild.spec.contracts.main.matching_dynamic_families"; +const DYNAMIC_FAMILIES_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; const MODEL_RESOURCE: &str = "model"; const SOURCE_RESOURCE: &str = "source"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index f4fddaa6b2..cd6fd74b01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -26,9 +26,8 @@ use crate::bindings::_helpers::discovery::guidance::sql_analysis_guidance; pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; -const SCENARIO_CONFIG_MODULE: &str = - "sqlbuild.spec.contracts.main.resolve_effective_scenario_config"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; +const SCENARIO_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index 516784814d..6500eb0a10 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -20,9 +20,8 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; const SCOPE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.scopes.constants"; const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; -const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; -const COLLECTION_RENDERING_MODULE: &str = - "sqlbuild.spec.contracts.main.resolve_effective_collection_rendering"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; +const COLLECTION_RENDERING_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.adapter.discovery.main.resolve_adapter"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; diff --git a/fensu.toml b/fensu.toml index bec5e7a491..d0485850c4 100644 --- a/fensu.toml +++ b/fensu.toml @@ -33,6 +33,11 @@ rule = "FFR309" path = "src/sqlbuild/cursor_algebra/__init__.py" reason = "The native cursor algebra in sqlbuild-cursor owns cursor behavior; this package only declares the cursor value models, sentinels and grain metadata callers import." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/spec/contracts/__init__.py" +reason = "Native bindings own configuration resolution; this package only declares the configuration models, constants and errors callers import." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/dag/types.py" diff --git a/python-boundary.toml b/python-boundary.toml index 7451ff087d..5f2b0979b5 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 58914 +remaining_lines = 58562 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 9268966f95..9572b5a0b8 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -300,6 +300,7 @@ from sqlbuild.spec.contracts.models import ( DbtConfig, DefaultsConfig, ExecutionLimitsConfig, + LoaderDestinationParts, LocalConfig, LocalDbtConfig, ProjectConfig, @@ -2492,11 +2493,30 @@ def effective_config_settings( settings: tuple[object, object], overrides: frozenset[str], names: list[str], / ) -> dict[str, object]: ... def yaml_entry_line_index(contents: str, /) -> list[tuple[str, int]]: ... - -class NamePattern: - def __init__(self, pattern: str) -> None: ... - def matches(self, column_name: str) -> bool: ... - +def resolve_target_name( + *, project_config: ProjectConfig, local_config: LocalConfig, selected_target: str | None +) -> str | None: ... +def resolve_target_config( + *, project_config: ProjectConfig, local_config: LocalConfig, target_name: str +) -> TargetConfig: ... +def resolve_effective_adapter_name( + *, project_config: ProjectConfig, local_config: LocalConfig +) -> str: ... +def resolve_effective_collection_rendering( + *, project_config: ProjectConfig, declaration_override: CollectionRendering | None +) -> CollectionRendering: ... +def resolve_effective_scenario_config( + *, project_config: ProjectConfig, local_config: LocalConfig +) -> ScenarioConfig: ... +def scenario_local_type_overrides_for_dialect( + *, scenario_config: ScenarioConfig, sql_analysis_dialect: str | None +) -> dict[str, str]: ... +def loader_destination_parts( + *, destination: str, default_database: str | None, default_schema: str | None +) -> LoaderDestinationParts: ... +def matching_dynamic_families( + *, families: tuple[SchemaDynamicColumnFamily, ...], column_name: str +) -> tuple[SchemaDynamicColumnFamily, ...]: ... def assert_no_unresolved_sql_markers(*, sql: str, context: str) -> None: ... def normalize_typed_sql_value( raw_value: object, typing: tuple[str, str | None], limits: tuple[int, int, int], / diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py index b48b7f91a2..441dba4b4b 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py @@ -6,7 +6,11 @@ from pathlib import Path from typing import Any -from sqlbuild._native import build_compiled_project, enforce_model_execution_limit +from sqlbuild._native import ( + build_compiled_project, + enforce_model_execution_limit, + resolve_target_config, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, @@ -61,7 +65,6 @@ CloneSourceEntries, ) from sqlbuild.executor.clone.types import CloneStatus -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.types import RetentionDecreasePolicy diff --git a/src/sqlbuild/cli/commands/_helpers/rules/command.py b/src/sqlbuild/cli/commands/_helpers/rules/command.py index b22b1aca8c..d158cfa71a 100644 --- a/src/sqlbuild/cli/commands/_helpers/rules/command.py +++ b/src/sqlbuild/cli/commands/_helpers/rules/command.py @@ -7,7 +7,11 @@ from dataclasses import replace from pathlib import Path -from sqlbuild._native import build_project_graph, discover_project_inputs +from sqlbuild._native import ( + build_project_graph, + discover_project_inputs, + resolve_effective_adapter_name, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands.constants import ( @@ -29,9 +33,6 @@ from sqlbuild.rule_engine.main.run_rules import run_rules from sqlbuild.rule_engine.main.skills import install_skills from sqlbuild.rule_engine.models import Rule, RulesConfig, RulesResult, RulesRunResult -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) def run_rules_command(request: RulesCommandRequest) -> int: diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py b/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py index ccb309ab74..155eaae005 100644 --- a/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py +++ b/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py @@ -4,14 +4,11 @@ from pathlib import Path -from sqlbuild._native import resolve_project_connection_config +from sqlbuild._native import resolve_effective_adapter_name, resolve_project_connection_config from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands.models import AdapterConnectionContext from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) def resolve_adapter_connection_context( diff --git a/src/sqlbuild/compiler/compile/main/_effective_target.py b/src/sqlbuild/compiler/compile/main/_effective_target.py index 7d07777ee6..bcbf17691f 100644 --- a/src/sqlbuild/compiler/compile/main/_effective_target.py +++ b/src/sqlbuild/compiler/compile/main/_effective_target.py @@ -2,9 +2,8 @@ from __future__ import annotations +from sqlbuild._native import resolve_target_config, resolve_target_name from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name from sqlbuild.spec.contracts.models import TargetConfig diff --git a/src/sqlbuild/compiler/compile/main/effective_config.py b/src/sqlbuild/compiler/compile/main/effective_config.py index 5bb306fe43..dfb2777f87 100644 --- a/src/sqlbuild/compiler/compile/main/effective_config.py +++ b/src/sqlbuild/compiler/compile/main/effective_config.py @@ -5,10 +5,10 @@ from sqlbuild._native import ( build_effective_connection, build_effective_vars, + resolve_target_config, + resolve_target_name, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name from sqlbuild.spec.contracts.models import TargetConfig diff --git a/src/sqlbuild/compiler/compile/main/effective_runtime.py b/src/sqlbuild/compiler/compile/main/effective_runtime.py index 7057413409..8f1a270910 100644 --- a/src/sqlbuild/compiler/compile/main/effective_runtime.py +++ b/src/sqlbuild/compiler/compile/main/effective_runtime.py @@ -5,10 +5,10 @@ from sqlbuild._native import ( build_effective_vars, resolve_run_id, + resolve_target_config, + resolve_target_name, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name from sqlbuild.spec.contracts.models import TargetConfig diff --git a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py index 1a4727e3a3..b1ca22b3d8 100644 --- a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py +++ b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py @@ -10,6 +10,8 @@ build_effective_vars, discover_project_inputs, rebind_declaration_scope, + resolve_effective_adapter_name, + resolve_effective_collection_rendering, ) from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.models import ( @@ -24,12 +26,6 @@ from sqlbuild.compiler.compile.types import TypedSqlValueRenderer from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( - resolve_effective_collection_rendering, -) def build_sql_expansion_context( diff --git a/src/sqlbuild/compiler/pipeline/_helpers/clone.py b/src/sqlbuild/compiler/pipeline/_helpers/clone.py index aa31ac89d8..9c0b7441cb 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/clone.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/clone.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild._native import build_compiled_project +from sqlbuild._native import build_compiled_project, resolve_target_config from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompileAnalysisSelection from sqlbuild.compiler.compile.types import CompiledProjectView @@ -14,7 +14,6 @@ ) from sqlbuild.compiler.planner.main.clone._clone import run_clone_planning from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.models import TargetConfig diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/schema.py b/src/sqlbuild/compiler/planner/_helpers/changes/schema.py index 6da7f7e8f8..c563f88afa 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/schema.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/schema.py @@ -4,12 +4,12 @@ from dataclasses import replace +from sqlbuild._native import matching_dynamic_families from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.type_system.main.types_equal import types_equal from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.planner.models import SchemaFinding from sqlbuild.compiler.planner.types import SchemaChangeKind, SchemaColumnSource -from sqlbuild.spec.contracts.main.matching_dynamic_families import matching_dynamic_families from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily _CASE_FOLDING_DIALECT: str = "snowflake" diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py b/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py index 086cd9a8fe..f921cf634e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py +++ b/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py @@ -2,10 +2,10 @@ from __future__ import annotations +from sqlbuild._native import loader_destination_parts from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.spec.contracts.main.loader_destination_parts import loader_destination_parts from sqlbuild.spec.contracts.models import LoaderDestinationParts, SourceEntry diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/retention.py b/src/sqlbuild/compiler/planner/_helpers/planning/retention.py index 8490daf239..2e508119f6 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/retention.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/retention.py @@ -6,6 +6,7 @@ from collections import defaultdict from dataclasses import replace +from sqlbuild._native import resolve_target_config from sqlbuild.adapter.contract.models import ( RelationInfo, RelationLookup, @@ -28,7 +29,6 @@ RetentionDirection, RetentionPlanPhase, ) -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention, TargetConfig from sqlbuild.spec.contracts.types import TableType diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py index 83d4c44c15..d7f40f15ea 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py @@ -5,7 +5,7 @@ import re from dataclasses import replace -from sqlbuild._native import render_project_var_text +from sqlbuild._native import render_project_var_text, resolve_target_config from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) @@ -28,7 +28,6 @@ ) from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceEntry, TargetConfig _CTX_PATTERN: re.Pattern[str] = re.compile(r"\$\{CTX:([^}]+)\}") diff --git a/src/sqlbuild/executor/pipeline/_helpers/scenario.py b/src/sqlbuild/executor/pipeline/_helpers/scenario.py index 1ecac28fe1..a7efed0c55 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/scenario.py +++ b/src/sqlbuild/executor/pipeline/_helpers/scenario.py @@ -18,6 +18,7 @@ from sqlbuild._native import ( NativeProgressProjector, current_native_progress_projector, + scenario_local_type_overrides_for_dialect, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import TablePromotionMode @@ -64,9 +65,6 @@ from sqlbuild.runtime.observability.classes.resource_attempt_lifecycle import ( ResourceAttemptLifecycle, ) -from sqlbuild.spec.contracts.main.scenario_local_type_overrides_for_dialect import ( - scenario_local_type_overrides_for_dialect, -) _DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") _STOP_POLL_SECONDS: float = 0.05 diff --git a/src/sqlbuild/executor/run/_helpers/validation/contracts.py b/src/sqlbuild/executor/run/_helpers/validation/contracts.py index b9a2dcfc4c..0836cc0b4b 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/contracts.py +++ b/src/sqlbuild/executor/run/_helpers/validation/contracts.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import matching_dynamic_families from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.adapter.type_system.main.types_equal import types_equal @@ -13,7 +14,6 @@ RUNTIME_CONTRACT_MISSING_DECLARATIONS_CODE, RUNTIME_CONTRACT_TYPE_MISMATCH_CODE, ) -from sqlbuild.spec.contracts.main.matching_dynamic_families import matching_dynamic_families from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py index 859ca77a78..9a3614fde1 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py @@ -9,6 +9,7 @@ apply_command_warehouse, render_dbt_execution_summary_footer, render_dbt_interop_plan, + resolve_target_config, ) from sqlbuild.cli.commands.main.dbt.dbt_sqlbuild_work import execute_dbt_sqlbuild_work from sqlbuild.cli.commands.main.execution.connection_progress import ( @@ -36,7 +37,6 @@ ) from sqlbuild.presentation.models import DisplayOptions from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.models import ExecutionLimitsConfig diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py index 10ab3b2a7f..3532cc35e7 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py @@ -16,6 +16,7 @@ load_dbt_manifest_index, plan_dbt_interop_command, resolve_dbt_executable, + resolve_effective_adapter_name, route_dbt_interop_args, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -50,9 +51,6 @@ DbtManifestIndex, ) from sqlbuild.integrations.dbt.types import DbtInteropCommand -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) def resolve_dbt_execution_invocation( diff --git a/src/sqlbuild/lint/_helpers/expansion.py b/src/sqlbuild/lint/_helpers/expansion.py index 1f409d2c2a..8bbbbeb57a 100644 --- a/src/sqlbuild/lint/_helpers/expansion.py +++ b/src/sqlbuild/lint/_helpers/expansion.py @@ -11,6 +11,7 @@ discover_project_inputs, expand_sql_with_spans, external_identifiers_for_headers, + resolve_effective_adapter_name, ) from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter @@ -46,9 +47,6 @@ PreparedLintFiles, ) from sqlbuild.lint.types import NativePreparedSql -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) _DEPENDENCY_INTRINSIC_PATTERN: re.Pattern[str] = re.compile( r"^__(?:ref|source)\s*\(", re.IGNORECASE diff --git a/src/sqlbuild/spec/contracts/_helpers/__init__.py b/src/sqlbuild/spec/contracts/_helpers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/spec/contracts/_helpers/dynamic_columns.py b/src/sqlbuild/spec/contracts/_helpers/dynamic_columns.py deleted file mode 100644 index 0a96e4bd31..0000000000 --- a/src/sqlbuild/spec/contracts/_helpers/dynamic_columns.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Shared matching helpers for dynamic column contract families.""" - -from __future__ import annotations - -from functools import lru_cache - -import sqlbuild._native as _native -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -def dynamic_family_accepts_name(*, family: SchemaDynamicColumnFamily, column_name: str) -> bool: - """Return whether a runtime column name belongs to a declared family.""" - - if family.name_pattern is None: - return True - return _compiled_name_pattern(family.name_pattern).matches(column_name) - - -def matching_dynamic_families( - *, families: tuple[SchemaDynamicColumnFamily, ...], column_name: str -) -> tuple[SchemaDynamicColumnFamily, ...]: - """Return every family whose optional naming constraint accepts a column.""" - - return tuple( - family - for family in families - if dynamic_family_accepts_name(family=family, column_name=column_name) - ) - - -@lru_cache(maxsize=256) -def _compiled_name_pattern(pattern: str) -> _native.NamePattern: - return _native.NamePattern(pattern) diff --git a/src/sqlbuild/spec/contracts/_helpers/project_config.py b/src/sqlbuild/spec/contracts/_helpers/project_config.py deleted file mode 100644 index bd620b96f9..0000000000 --- a/src/sqlbuild/spec/contracts/_helpers/project_config.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Project configuration resolution implementations.""" - -from __future__ import annotations - -from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - ScenarioConfig, - ScenarioSnapshotLimitsConfig, -) -from sqlbuild.sql_values.types import CollectionRendering - - -def resolve_effective_collection_rendering( - *, - project_config: ProjectConfig, - declaration_override: CollectionRendering | None, -) -> CollectionRendering: - """Resolve declaration rendering over project configuration.""" - - if declaration_override is not None: - return declaration_override - return project_config.constants.collection_rendering - - -def resolve_effective_adapter_name( - *, project_config: ProjectConfig, local_config: LocalConfig -) -> str: - """Resolve the effective adapter name, allowing local override.""" - - if local_config.adapter is not None: - return local_config.adapter - return project_config.adapter - - -def resolve_effective_scenario_config( - *, project_config: ProjectConfig, local_config: LocalConfig -) -> ScenarioConfig: - """Resolve scenario config, allowing local overrides to replace project rules.""" - - local_type_overrides: dict[str, dict[str, str]] = { - dialect: dict(rules) - for dialect, rules in project_config.scenario.local_type_overrides.items() - } - dialect: str - rules: dict[str, str] - for dialect, rules in local_config.scenario.local_type_overrides.items(): - local_type_overrides.setdefault(dialect, {}).update(rules) - project_limits: ScenarioSnapshotLimitsConfig = project_config.scenario.snapshot_limits - local_limits: ScenarioSnapshotLimitsConfig = local_config.scenario.snapshot_limits - return ScenarioConfig( - run_namespace=( - local_config.scenario.run_namespace - if local_config.scenario.run_namespace is not None - else project_config.scenario.run_namespace - ), - local_type_overrides=local_type_overrides, - snapshot_limits=ScenarioSnapshotLimitsConfig( - max_rows_per_relation=_resolve_optional_int_override( - project_value=project_limits.max_rows_per_relation, - local_value=local_limits.max_rows_per_relation, - ), - max_total_rows=_resolve_optional_int_override( - project_value=project_limits.max_total_rows, - local_value=local_limits.max_total_rows, - ), - max_bytes_per_relation=_resolve_optional_int_override( - project_value=project_limits.max_bytes_per_relation, - local_value=local_limits.max_bytes_per_relation, - ), - max_total_bytes=_resolve_optional_int_override( - project_value=project_limits.max_total_bytes, - local_value=local_limits.max_total_bytes, - ), - ), - ) - - -def scenario_local_type_overrides_for_dialect( - *, scenario_config: ScenarioConfig, sql_analysis_dialect: str | None -) -> dict[str, str]: - """Return global and dialect-specific scenario local type override rules.""" - - overrides: dict[str, str] = dict(scenario_config.local_type_overrides.get("*", {})) - if sql_analysis_dialect is not None: - overrides.update(scenario_config.local_type_overrides.get(sql_analysis_dialect, {})) - return overrides - - -def _resolve_optional_int_override( - *, project_value: int | None, local_value: int | None -) -> int | None: - return local_value if local_value is not None else project_value diff --git a/src/sqlbuild/spec/contracts/_helpers/targets.py b/src/sqlbuild/spec/contracts/_helpers/targets.py deleted file mode 100644 index f35d453908..0000000000 --- a/src/sqlbuild/spec/contracts/_helpers/targets.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Target selection and target merging, resolved natively over the loaded config objects.""" - -from __future__ import annotations - -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.models import ( - ClonePolicy, - ExecutionLimitsConfig, - LocalConfig, - ProjectConfig, - TargetConfig, - TargetDiffConfig, - TargetWarehousesConfig, -) - -_GROUPS: dict[str, type[object]] = { - "clone": ClonePolicy, - "execution_limits": ExecutionLimitsConfig, - "diff": TargetDiffConfig, - "warehouses": TargetWarehousesConfig, -} - - -def resolve_target_name( - *, - project_config: ProjectConfig, - local_config: LocalConfig, - selected_target: str | None, -) -> str | None: - """Resolve the effective target name.""" - - name, error = _native.effective_config_target_name( - project_config, local_config, selected_target - ) - if error is not None: - raise SpecConfigError(error) - return name - - -def resolve_target_config( - *, - project_config: ProjectConfig, - local_config: LocalConfig, - target_name: str, -) -> TargetConfig: - """Merge project target config with local developer overrides.""" - - rows, error = _native.effective_config_target_fields( - (project_config, local_config), target_name, TargetConfig() - ) - if rows is None: - raise SpecConfigError(error or "") - values: dict[str, Any] = {} - grouped: dict[str, dict[str, Any]] = {group: {} for group in _GROUPS} - for group, name, value in rows: - (values if group is None else grouped[group])[name] = value - nested: dict[str, Any] = {group: kind(**grouped[group]) for group, kind in _GROUPS.items()} - return TargetConfig(**values, **nested) diff --git a/src/sqlbuild/spec/contracts/constants.py b/src/sqlbuild/spec/contracts/constants.py index 917e001590..658c3c4a6a 100644 --- a/src/sqlbuild/spec/contracts/constants.py +++ b/src/sqlbuild/spec/contracts/constants.py @@ -4,8 +4,6 @@ from sqlbuild.spec.contracts.models import SeedCsvSettings -LOADER_SCHEMA_TABLE_PART_COUNT: int = 2 -LOADER_QUALIFIED_TABLE_PART_COUNT: int = 3 DEFAULT_SEED_CSV_SETTINGS: SeedCsvSettings = SeedCsvSettings() CURSOR_POLICY_DISABLED: str = "disabled" ZERO_DAY_CURSOR_DURATION: str = "0d" diff --git a/src/sqlbuild/spec/contracts/main/__init__.py b/src/sqlbuild/spec/contracts/main/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/spec/contracts/main/loader_destination_parts.py b/src/sqlbuild/spec/contracts/main/loader_destination_parts.py deleted file mode 100644 index 7abf45a943..0000000000 --- a/src/sqlbuild/spec/contracts/main/loader_destination_parts.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Loader destination parsing contract.""" - -from __future__ import annotations - -from sqlbuild.spec.contracts.constants import ( - LOADER_QUALIFIED_TABLE_PART_COUNT, - LOADER_SCHEMA_TABLE_PART_COUNT, -) -from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.models import LoaderDestinationParts - - -def loader_destination_parts( - *, destination: str, default_database: str | None, default_schema: str | None -) -> LoaderDestinationParts: - """Parse a one-, two-, or three-part loader destination.""" - - parts: tuple[str, ...] = tuple(destination.split(".")) - if not 1 <= len(parts) <= LOADER_QUALIFIED_TABLE_PART_COUNT or any( - not part.strip() for part in parts - ): - raise SpecConfigError( - f"Invalid loader destination '{destination}': expected 1 to 3 non-empty " - "dot-separated parts (table, schema.table, or database.schema.table)" - ) - if len(parts) == 1: - return LoaderDestinationParts(default_database, default_schema, parts[0]) - if len(parts) == LOADER_SCHEMA_TABLE_PART_COUNT: - return LoaderDestinationParts(default_database, parts[0], parts[1]) - return LoaderDestinationParts(parts[0], parts[1], parts[2]) diff --git a/src/sqlbuild/spec/contracts/main/matching_dynamic_families.py b/src/sqlbuild/spec/contracts/main/matching_dynamic_families.py deleted file mode 100644 index 7f1823b328..0000000000 --- a/src/sqlbuild/spec/contracts/main/matching_dynamic_families.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public compiler entry point for dynamic-family name matching.""" - -from sqlbuild.spec.contracts._helpers.dynamic_columns import matching_dynamic_families as _match -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -def matching_dynamic_families( - *, families: tuple[SchemaDynamicColumnFamily, ...], column_name: str -) -> tuple[SchemaDynamicColumnFamily, ...]: - """Return declared dynamic families accepting a physical column name.""" - - return _match(families=families, column_name=column_name) diff --git a/src/sqlbuild/spec/contracts/main/resolve_effective_adapter_name.py b/src/sqlbuild/spec/contracts/main/resolve_effective_adapter_name.py deleted file mode 100644 index 94e0679659..0000000000 --- a/src/sqlbuild/spec/contracts/main/resolve_effective_adapter_name.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Public effective adapter name resolution operation.""" - -from sqlbuild.spec.contracts._helpers.project_config import ( - resolve_effective_adapter_name as _resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -def resolve_effective_adapter_name( - *, project_config: ProjectConfig, local_config: LocalConfig -) -> str: - """Resolve the effective adapter name, allowing local override.""" - - return _resolve_effective_adapter_name( - project_config=project_config, - local_config=local_config, - ) diff --git a/src/sqlbuild/spec/contracts/main/resolve_effective_collection_rendering.py b/src/sqlbuild/spec/contracts/main/resolve_effective_collection_rendering.py deleted file mode 100644 index 0a978fdcc3..0000000000 --- a/src/sqlbuild/spec/contracts/main/resolve_effective_collection_rendering.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public effective collection rendering resolution operation.""" - -from sqlbuild.spec.contracts._helpers.project_config import ( - resolve_effective_collection_rendering as _resolve_effective_collection_rendering, -) -from sqlbuild.spec.contracts.models import ProjectConfig -from sqlbuild.sql_values.types import CollectionRendering - - -def resolve_effective_collection_rendering( - *, - project_config: ProjectConfig, - declaration_override: CollectionRendering | None, -) -> CollectionRendering: - """Resolve declaration rendering over project configuration.""" - - return _resolve_effective_collection_rendering( - project_config=project_config, - declaration_override=declaration_override, - ) diff --git a/src/sqlbuild/spec/contracts/main/resolve_effective_scenario_config.py b/src/sqlbuild/spec/contracts/main/resolve_effective_scenario_config.py deleted file mode 100644 index 277516fc40..0000000000 --- a/src/sqlbuild/spec/contracts/main/resolve_effective_scenario_config.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Public effective scenario configuration resolution operation.""" - -from sqlbuild.spec.contracts._helpers.project_config import ( - resolve_effective_scenario_config as _resolve_effective_scenario_config, -) -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, ScenarioConfig - - -def resolve_effective_scenario_config( - *, project_config: ProjectConfig, local_config: LocalConfig -) -> ScenarioConfig: - """Resolve scenario config, allowing local overrides to replace project rules.""" - - return _resolve_effective_scenario_config( - project_config=project_config, - local_config=local_config, - ) diff --git a/src/sqlbuild/spec/contracts/main/resolve_target_config.py b/src/sqlbuild/spec/contracts/main/resolve_target_config.py deleted file mode 100644 index 341799dc4f..0000000000 --- a/src/sqlbuild/spec/contracts/main/resolve_target_config.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Public target configuration resolution operation.""" - -from sqlbuild.spec.contracts._helpers.targets import ( - resolve_target_config as _resolve_target_config, -) -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, TargetConfig - - -def resolve_target_config( - *, - project_config: ProjectConfig, - local_config: LocalConfig, - target_name: str, -) -> TargetConfig: - """Merge project target config with local developer overrides.""" - - return _resolve_target_config( - project_config=project_config, - local_config=local_config, - target_name=target_name, - ) diff --git a/src/sqlbuild/spec/contracts/main/resolve_target_name.py b/src/sqlbuild/spec/contracts/main/resolve_target_name.py deleted file mode 100644 index f200723b98..0000000000 --- a/src/sqlbuild/spec/contracts/main/resolve_target_name.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Public target name resolution operation.""" - -from sqlbuild.spec.contracts._helpers.targets import resolve_target_name as _resolve_target_name -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -def resolve_target_name( - *, - project_config: ProjectConfig, - local_config: LocalConfig, - selected_target: str | None, -) -> str | None: - """Resolve the effective target name.""" - - return _resolve_target_name( - project_config=project_config, - local_config=local_config, - selected_target=selected_target, - ) diff --git a/src/sqlbuild/spec/contracts/main/scenario_local_type_overrides_for_dialect.py b/src/sqlbuild/spec/contracts/main/scenario_local_type_overrides_for_dialect.py deleted file mode 100644 index d397a39e27..0000000000 --- a/src/sqlbuild/spec/contracts/main/scenario_local_type_overrides_for_dialect.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Public scenario local type override selection operation.""" - -from sqlbuild.spec.contracts._helpers.project_config import ( - scenario_local_type_overrides_for_dialect as _scenario_local_type_overrides_for_dialect, -) -from sqlbuild.spec.contracts.models import ScenarioConfig - - -def scenario_local_type_overrides_for_dialect( - *, scenario_config: ScenarioConfig, sql_analysis_dialect: str | None -) -> dict[str, str]: - """Return global and dialect-specific scenario local type override rules.""" - - return _scenario_local_type_overrides_for_dialect( - scenario_config=scenario_config, - sql_analysis_dialect=sql_analysis_dialect, - ) diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py index 9607dae950..e28c5e327a 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -21,6 +21,7 @@ expand_sql_macros, mask_capture_noise, render_stage_capture, + resolve_effective_collection_rendering, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -37,9 +38,6 @@ from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE from sqlbuild.compiler.scopes.models import ResourceIdentity from sqlbuild.compiler.scopes.types import ResourceKind -from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( - resolve_effective_collection_rendering, -) MACRO_CALL_LOG_ENV_VAR: str = "SQLBUILD_TEST_MACRO_CALL_LOG" _RUN_ID_PATTERN: re.Pattern[str] = re.compile(r'"run_id": "[^"]*"') diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index 20fed3b8c5..11c98422f9 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -12,15 +12,17 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import build_compile_inputs, discover_project_inputs, parse_audit_instances +from sqlbuild._native import ( + build_compile_inputs, + discover_project_inputs, + parse_audit_instances, + resolve_effective_collection_rendering, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileAdapterContext from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile -from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( - resolve_effective_collection_rendering, -) from sqlbuild.spec.contracts.models import SchemaModelEntry, SourceLocation _TEXTS: tuple[object, ...] = ( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py index a3207baee3..13b69a940a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py @@ -87,7 +87,7 @@ def test_given_pending_run_when_showing_detail_then_persisted_summary_is_refresh lambda **kwargs: SimpleNamespace(project_config=object(), local_config=object()), ) monkeypatch.setattr( - "sqlbuild.spec.contracts.main.resolve_effective_adapter_name.resolve_effective_adapter_name", + "sqlbuild._native.resolve_effective_adapter_name", lambda **kwargs: "snowflake", ) monkeypatch.setattr( @@ -191,7 +191,7 @@ def test_given_useful_partial_when_refresh_degrades_then_persisted_summary_is_pr lambda **kwargs: SimpleNamespace(project_config=object(), local_config=object()), ) monkeypatch.setattr( - "sqlbuild.spec.contracts.main.resolve_effective_adapter_name.resolve_effective_adapter_name", + "sqlbuild._native.resolve_effective_adapter_name", lambda **kwargs: "snowflake", ) monkeypatch.setattr( @@ -274,7 +274,7 @@ def test_given_persisted_ledger_failure_when_refresh_has_no_coverage_then_failur lambda **kwargs: SimpleNamespace(project_config=object(), local_config=object()), ) monkeypatch.setattr( - "sqlbuild.spec.contracts.main.resolve_effective_adapter_name.resolve_effective_adapter_name", + "sqlbuild._native.resolve_effective_adapter_name", lambda **kwargs: "snowflake", ) monkeypatch.setattr( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py index 4544a713e1..34f8c5f0c9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py @@ -5,12 +5,10 @@ import pytest +from sqlbuild._native import resolve_effective_adapter_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( ExpectedMessageTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py index 131bee4ac8..f5a4a72bee 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py @@ -6,12 +6,11 @@ from pathlib import Path from typing import Any, Protocol, cast -from sqlbuild._native import NativeProgressProjector +from sqlbuild._native import NativeProgressProjector, loader_destination_parts from sqlbuild.cli.entry.models import ( CliEntrypointHandlers, ) from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.main.loader_destination_parts import loader_destination_parts class JsonOutputRequest(Protocol): diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_main.py b/tests/unit/src/sqlbuild/compiler/compile/test_main.py index fe89344a7f..61850194b7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_main.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild._native import build_compile_inputs, discover_project_inputs +from sqlbuild._native import build_compile_inputs, discover_project_inputs, resolve_target_config from sqlbuild.compiler.compile.models import ( CompileModelConfig, ) @@ -19,7 +19,6 @@ ) from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.models import ( AuthoredTimeTravelRetention, ClonePolicy, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_diff.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_diff.py index d0f29c74d3..b8ce19c597 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_diff.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_diff.py @@ -6,8 +6,8 @@ import pytest +from sqlbuild._native import resolve_target_config from sqlbuild.compiler.discovery.exceptions import ProjectConfigError -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( TargetDiffConfigErrorTestCase, TargetDiffConfigTestCase, diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py index 79b1ccf7cd..77842a9979 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py @@ -2,9 +2,7 @@ import pytest -from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( - resolve_effective_collection_rendering, -) +from sqlbuild._native import resolve_effective_collection_rendering from sqlbuild.spec.contracts.models import ConstantsConfig, ProjectConfig from sqlbuild.sql_values.types import CollectionRendering from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py index f324441d7c..359873a767 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import loader_destination_parts from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.main.loader_destination_parts import loader_destination_parts from sqlbuild.spec.contracts.models import LoaderDestinationParts from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( InvalidLoaderDestinationTestCase, diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py index d54a6514fe..ce278ee9a6 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py @@ -6,12 +6,9 @@ import pytest -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import discover_project_inputs, resolve_effective_scenario_config from sqlbuild.compiler.discovery.exceptions import ProjectConfigError from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.main.resolve_effective_scenario_config import ( - resolve_effective_scenario_config, -) from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( InvalidScenarioNamespaceConfigTestCase, ScenarioNamespaceConfigTestCase, diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py index c8a5971a79..f3ce3f24f0 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py @@ -2,8 +2,8 @@ import pytest +from sqlbuild._native import resolve_target_config from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.models import ( AuthoredTimeTravelRetention, ClonePolicy, From a59606c950bb7203d6db316bf68e9e22d2c5a045 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 04:24:03 +0100 Subject: [PATCH 14/40] refactor: prepare clone pipelines natively Origin and destination compilation, source deferral targets and the ClonePipelineResult are assembled by the native run_clone_pipeline_with_options binding, which the clone command calls directly. --- .../bindings/_helpers/clone/clone_command.rs | 25 ++- .../bindings/_helpers/clone/clone_pipeline.rs | 167 ++++++++++++++++++ .../src/bindings/_helpers/clone/mod.rs | 1 + .../bindings/_helpers/clone/registration.rs | 5 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 13 ++ .../_helpers/build_planning/defer_clone.py | 2 +- .../compiler/pipeline/_helpers/clone.py | 148 ---------------- src/sqlbuild/compiler/pipeline/main/_clone.py | 42 ----- .../pipeline/main/clone_with_options.py | 36 ---- .../compiler/pipeline/_helpers/test_clone.py | 6 +- 11 files changed, 205 insertions(+), 242 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs delete mode 100644 src/sqlbuild/compiler/pipeline/_helpers/clone.py delete mode 100644 src/sqlbuild/compiler/pipeline/main/_clone.py delete mode 100644 src/sqlbuild/compiler/pipeline/main/clone_with_options.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index 32d0ebd4ee..d131b3336b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -21,6 +21,7 @@ use sqlbuild_core::clone_text::models::CloneItemText; use crate::bindings::_helpers::clone::clone_execution::{ destination_entry_count, execute_clone_plan, resource_types_by_name, }; +use crate::bindings::_helpers::clone::clone_pipeline::{ClonePipeline, prepared_clone_pipeline}; use crate::bindings::_helpers::clone::clone_progress::{clone_item_text, write_line}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; @@ -33,7 +34,6 @@ const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; -const CLONE_PIPELINE_MODULE: &str = "sqlbuild.compiler.pipeline.main._clone"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const NO_DESTINATION: (&str, &str) = ( @@ -265,21 +265,28 @@ fn prepare<'py>( &imported(py, PIPELINE_MODELS, "ClonePipelineConnection")?, &[("config", config.clone()), ("handle", connection.clone())], )?; - let pipeline: Bound<'py, PyAny> = call_with( - &imported(py, CLONE_PIPELINE_MODULE, "run_clone_pipeline")?, + let options: Bound<'py, PyAny> = call_with( + &imported(py, PIPELINE_MODELS, "ClonePipelineOptions")?, &[ - ("discovered_inputs", invocation.discovered.clone()), - ("adapter", invocation.adapter.clone()), - ("origin_target_name", request.getattr("origin_target_name")?), - ("destination_target_name", text(py, &invocation.destination)), ("no_sql_validation", request.getattr("no_sql_validation")?), ("select", request.getattr("select")?), ("exclude", request.getattr("exclude")?), ("cli_vars", request.getattr("cli_vars")?), - ("destination_connection", destination_connection), - ("external_sql_reference_resolver", resolver), ], )?; + let pipeline: Bound<'py, PyAny> = prepared_clone_pipeline( + &ClonePipeline { + discovered: invocation.discovered.clone(), + adapter: invocation.adapter.clone(), + connection: destination_connection, + options, + resolver, + }, + ( + &request.getattr("origin_target_name")?, + &text(py, &invocation.destination), + ), + )?; progress.call_method1( "on_progress", (format!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs new file mode 100644 index 0000000000..1677450c25 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs @@ -0,0 +1,167 @@ +//! Compile the origin and destination environments of a clone and plan it. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; +const CLONE_PLANNING_MODULE: &str = "sqlbuild.compiler.planner.main.clone._clone"; +/// The `ClonePipelineResult` fields after the two projects, in `run_clone_planning` order. +const PLANNED_FIELDS: [&str; 8] = [ + "clone_plan", + "destination_source_entries", + "destination_model_entries", + "destination_seed_entries", + "destination_function_entries", + "origin_model_entries", + "origin_seed_entries", + "origin_source_entries", +]; + +/// The discovered project, adapter, destination connection, options and reference resolver. +pub(crate) struct ClonePipeline<'py> { + pub(crate) discovered: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) connection: Bound<'py, PyAny>, + pub(crate) options: Bound<'py, PyAny>, + pub(crate) resolver: Bound<'py, PyAny>, +} + +/// The compiled project of one target environment. +fn compiled_environment<'py>( + pipeline: &ClonePipeline<'py>, + target_name: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = target_name.py(); + let options: &Bound<'py, PyAny> = &pipeline.options; + call_with( + &imported(py, NATIVE_MODULE, "build_compiled_project")?, + &[ + ("discovered_inputs", pipeline.discovered.clone()), + ("adapter", pipeline.adapter.clone()), + ("selected_target", target_name.clone()), + ( + "resolved_connection", + pipeline.connection.getattr("config")?, + ), + ("no_sql_validation", options.getattr("no_sql_validation")?), + ( + "analysis_selection", + call_with( + &imported(py, COMPILE_MODELS_MODULE, "CompileAnalysisSelection")?, + &[("no_cache", options.getattr("no_cache")?)], + )?, + ), + ("cli_vars", options.getattr("cli_vars")?), + ("external_sql_reference_resolver", pipeline.resolver.clone()), + ], + ) +} + +/// The target whose sources a target reads: its `defer_sources_to`, or itself. +fn read_target_name<'py>( + discovered: &Bound<'py, PyAny>, + target_name: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = target_name.py(); + let deferred: Bound<'py, PyAny> = resolve_target_config( + py, + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, + target_name, + )? + .getattr("defer_sources_to")?; + if deferred.is_truthy()? { + return Ok(deferred); + } + Ok(target_name.clone()) +} + +/// The project its sources are read from: `project` itself unless sources defer elsewhere. +fn source_project<'py>( + pipeline: &ClonePipeline<'py>, + project: &Bound<'py, PyAny>, + target_name: &Bound<'py, PyAny>, +) -> PyResult> { + let read_target: Bound<'py, PyAny> = read_target_name(&pipeline.discovered, target_name)?; + if read_target.eq(target_name)? { + return Ok(project.clone()); + } + compiled_environment(pipeline, &read_target) +} + +/// Compile both environments, plan the clone and return the `ClonePipelineResult`. +pub(crate) fn prepared_clone_pipeline<'py>( + pipeline: &ClonePipeline<'py>, + (origin_target, destination_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = origin_target.py(); + let origin_project: Bound<'py, PyAny> = compiled_environment(pipeline, origin_target)?; + let destination_project: Bound<'py, PyAny> = + compiled_environment(pipeline, destination_target)?; + let origin_source_project: Bound<'py, PyAny> = + source_project(pipeline, &origin_project, origin_target)?; + let destination_source_project: Bound<'py, PyAny> = + source_project(pipeline, &destination_project, destination_target)?; + let planned: Bound<'py, PyTuple> = call_with( + &imported(py, CLONE_PLANNING_MODULE, "run_clone_planning")?, + &[ + ("project", destination_project.clone()), + ("select", pipeline.options.getattr("select")?), + ("exclude", pipeline.options.getattr("exclude")?), + ("adapter", pipeline.adapter.clone()), + ("connection", pipeline.connection.getattr("handle")?), + ("origin_project", origin_project.clone()), + ("origin_source_project", origin_source_project), + ("destination_source_project", destination_source_project), + ], + )? + .downcast_into()?; + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("origin_project", origin_project), + ("destination_project", destination_project), + ]; + for (index, name) in PLANNED_FIELDS.iter().enumerate() { + fields.push((*name, planned.get_item(index)?)); + } + call_with( + &imported(py, PIPELINE_MODELS_MODULE, "ClonePipelineResult")?, + &fields, + ) +} + +/// Run a clone pipeline with explicit compilation controls. +#[pyfunction] +#[pyo3(signature = (**request))] +fn run_clone_pipeline_with_options<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request: Request<'_, 'py> = Request::new("run_clone_pipeline_with_options", request)?; + let py: Python<'py> = request.py(); + prepared_clone_pipeline( + &ClonePipeline { + discovered: request.get("discovered_inputs")?, + adapter: request.get("adapter")?, + connection: request.get("destination_connection")?, + options: request.get("options")?, + resolver: request + .optional("external_sql_reference_resolver")? + .unwrap_or_else(|| py.None().into_bound(py)), + }, + ( + &request.get("origin_target_name")?, + &request.get("destination_target_name")?, + ), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_clone_pipeline_with_options, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/mod.rs index 6ac5b67d26..9a3293f7cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod clone_command; pub(crate) mod clone_execution; +pub(crate) mod clone_pipeline; pub(crate) mod clone_progress; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs index 9df257097e..bca0112872 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/registration.rs @@ -2,8 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::clone::clone_command; +use crate::bindings::_helpers::clone::{clone_command, clone_pipeline}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - clone_command::register(module) + clone_command::register(module)?; + clone_pipeline::register(module) } diff --git a/python-boundary.toml b/python-boundary.toml index 5f2b0979b5..d7f034c0e4 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 58562 +remaining_lines = 58336 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 9572b5a0b8..318bf7c7da 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -169,6 +169,9 @@ from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullabili from sqlbuild.compiler.migrations.models import MigrationRelation from sqlbuild.compiler.migrations.types import OldNameViewEventType from sqlbuild.compiler.pipeline.models import ( + ClonePipelineConnection, + ClonePipelineOptions, + ClonePipelineResult, CompiledProjectPhaseResult, CompilePipelineOptions, CompilePipelineResult, @@ -2511,6 +2514,16 @@ def resolve_effective_scenario_config( def scenario_local_type_overrides_for_dialect( *, scenario_config: ScenarioConfig, sql_analysis_dialect: str | None ) -> dict[str, str]: ... +def run_clone_pipeline_with_options( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + origin_target_name: str, + destination_target_name: str, + destination_connection: ClonePipelineConnection, + options: ClonePipelineOptions, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, +) -> ClonePipelineResult: ... def loader_destination_parts( *, destination: str, default_database: str | None, default_schema: str | None ) -> LoaderDestinationParts: ... diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py index 441dba4b4b..e5d80c7ce0 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py @@ -10,6 +10,7 @@ build_compiled_project, enforce_model_execution_limit, resolve_target_config, + run_clone_pipeline_with_options, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.planning.external_refs import ( @@ -34,7 +35,6 @@ CompiledResourceType, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.clone_with_options import run_clone_pipeline_with_options from sqlbuild.compiler.pipeline.main.unit_test_selection import split_unit_test_selectors from sqlbuild.compiler.pipeline.models import ( ClonePipelineConnection, diff --git a/src/sqlbuild/compiler/pipeline/_helpers/clone.py b/src/sqlbuild/compiler/pipeline/_helpers/clone.py deleted file mode 100644 index 9c0b7441cb..0000000000 --- a/src/sqlbuild/compiler/pipeline/_helpers/clone.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Clone pipeline assembly helpers.""" - -from __future__ import annotations - -from sqlbuild._native import build_compiled_project, resolve_target_config -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompileAnalysisSelection -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ( - ClonePipelineConnection, - ClonePipelineOptions, - ClonePipelineResult, -) -from sqlbuild.compiler.planner.main.clone._clone import run_clone_planning -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.spec.contracts.models import TargetConfig - - -def prepare_clone_pipeline( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - origin_target_name: str, - destination_target_name: str, - destination_connection: ClonePipelineConnection, - options: ClonePipelineOptions, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, -) -> ClonePipelineResult: - origin_project: CompiledProjectView = _compile_project_for_environment( - discovered_inputs=discovered_inputs, - adapter=adapter, - target_name=origin_target_name, - resolved_connection=destination_connection.config, - no_sql_validation=options.no_sql_validation, - no_cache=options.no_cache, - cli_vars=options.cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - destination_project: CompiledProjectView = _compile_project_for_environment( - discovered_inputs=discovered_inputs, - adapter=adapter, - target_name=destination_target_name, - resolved_connection=destination_connection.config, - no_sql_validation=options.no_sql_validation, - no_cache=options.no_cache, - cli_vars=options.cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - origin_read_target_name: str = _read_target_name( - discovered_inputs=discovered_inputs, - target_name=origin_target_name, - ) - destination_read_target_name: str = _read_target_name( - discovered_inputs=discovered_inputs, - target_name=destination_target_name, - ) - origin_source_project: CompiledProjectView = ( - origin_project - if origin_read_target_name == origin_target_name - else _compile_project_for_environment( - discovered_inputs=discovered_inputs, - adapter=adapter, - target_name=origin_read_target_name, - resolved_connection=destination_connection.config, - no_sql_validation=options.no_sql_validation, - no_cache=options.no_cache, - cli_vars=options.cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - ) - destination_source_project: CompiledProjectView = ( - destination_project - if destination_read_target_name == destination_target_name - else _compile_project_for_environment( - discovered_inputs=discovered_inputs, - adapter=adapter, - target_name=destination_read_target_name, - resolved_connection=destination_connection.config, - no_sql_validation=options.no_sql_validation, - no_cache=options.no_cache, - cli_vars=options.cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - ) - ( - clone_plan, - destination_source_entries, - destination_model_entries, - destination_seed_entries, - destination_function_entries, - origin_model_entries, - origin_seed_entries, - origin_source_entries, - ) = run_clone_planning( - project=destination_project, - select=options.select, - exclude=options.exclude, - adapter=adapter, - connection=destination_connection.handle, - origin_project=origin_project, - origin_source_project=origin_source_project, - destination_source_project=destination_source_project, - ) - return ClonePipelineResult( - origin_project=origin_project, - destination_project=destination_project, - clone_plan=clone_plan, - destination_source_entries=destination_source_entries, - destination_model_entries=destination_model_entries, - destination_seed_entries=destination_seed_entries, - destination_function_entries=destination_function_entries, - origin_model_entries=origin_model_entries, - origin_seed_entries=origin_seed_entries, - origin_source_entries=origin_source_entries, - ) - - -def _compile_project_for_environment( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - target_name: str, - resolved_connection: dict[str, object], - no_sql_validation: bool, - no_cache: bool, - cli_vars: dict[str, object] | None, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None, -) -> CompiledProjectView: - return build_compiled_project( - discovered_inputs=discovered_inputs, - adapter=adapter, - selected_target=target_name, - resolved_connection=resolved_connection, - no_sql_validation=no_sql_validation, - analysis_selection=CompileAnalysisSelection(no_cache=no_cache), - cli_vars=cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - - -def _read_target_name(*, discovered_inputs: DiscoveredProjectInputs, target_name: str) -> str: - target_config: TargetConfig = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ) - return target_config.defer_sources_to or target_name diff --git a/src/sqlbuild/compiler/pipeline/main/_clone.py b/src/sqlbuild/compiler/pipeline/main/_clone.py deleted file mode 100644 index 61bfa037b1..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/_clone.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Public clone pipeline entrypoint.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.clone import prepare_clone_pipeline -from sqlbuild.compiler.pipeline.models import ( - ClonePipelineConnection, - ClonePipelineOptions, - ClonePipelineResult, -) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver - - -def run_clone_pipeline( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - origin_target_name: str, - destination_target_name: str, - no_sql_validation: bool = False, - select: tuple[str, ...] = (), - exclude: tuple[str, ...] = (), - cli_vars: dict[str, object] | None = None, - destination_connection: ClonePipelineConnection, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, -) -> ClonePipelineResult: - return prepare_clone_pipeline( - discovered_inputs=discovered_inputs, - adapter=adapter, - origin_target_name=origin_target_name, - destination_target_name=destination_target_name, - destination_connection=destination_connection, - external_sql_reference_resolver=external_sql_reference_resolver, - options=ClonePipelineOptions( - no_sql_validation=no_sql_validation, - select=select, - exclude=exclude, - cli_vars=cli_vars, - ), - ) diff --git a/src/sqlbuild/compiler/pipeline/main/clone_with_options.py b/src/sqlbuild/compiler/pipeline/main/clone_with_options.py deleted file mode 100644 index 8d8ffaa098..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/clone_with_options.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Clone pipeline entrypoint with typed compilation options.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.clone import prepare_clone_pipeline -from sqlbuild.compiler.pipeline.models import ( - ClonePipelineConnection, - ClonePipelineOptions, - ClonePipelineResult, -) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver - - -def run_clone_pipeline_with_options( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - origin_target_name: str, - destination_target_name: str, - destination_connection: ClonePipelineConnection, - options: ClonePipelineOptions, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, -) -> ClonePipelineResult: - """Run a clone pipeline with explicit compilation controls.""" - - return prepare_clone_pipeline( - discovered_inputs=discovered_inputs, - adapter=adapter, - origin_target_name=origin_target_name, - destination_target_name=destination_target_name, - destination_connection=destination_connection, - options=options, - external_sql_reference_resolver=external_sql_reference_resolver, - ) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py index 7ddaecfa59..b83e6509d0 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py @@ -2,9 +2,9 @@ import pytest +from sqlbuild._native import run_clone_pipeline_with_options from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.clone import prepare_clone_pipeline from sqlbuild.compiler.pipeline.models import ( ClonePipelineConnection, ClonePipelineOptions, @@ -54,7 +54,7 @@ def test_given_clone_target_without_schema_when_compiling_then_fails_before_plan ValueError, match=f"Named target '{test_case.expected_target_name}' must explicitly set schema", ): - prepare_clone_pipeline( + run_clone_pipeline_with_options( discovered_inputs=discovered_inputs, adapter=PostgresAdapter(), origin_target_name="prod", @@ -97,7 +97,7 @@ def test_given_clone_targets_when_compiling_then_all_projects_use_destination_co local_config=LocalConfig(), ) - result: ClonePipelineResult = prepare_clone_pipeline( + result: ClonePipelineResult = run_clone_pipeline_with_options( discovered_inputs=discovered_inputs, adapter=PostgresAdapter(), origin_target_name="prod", From f72917de8bbfb9a19da4a173a3c883c0728ada5e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 04:30:34 +0100 Subject: [PATCH 15/40] refactor: build native declaration contexts in the Rust glue The scope-lookup handoff to NativeDeclarationContexts moves into the Rust declaration resolver, removing the last model_loop module. Loader namespace and warehouse templates call the Rust template expansion directly. --- .../_helpers/command_runtime/warehouses.rs | 43 ++-- .../_helpers/compile_driver/targets.rs | 30 ++- .../_helpers/render/declaration_values.rs | 194 ++++++++++++++++-- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 5 + src/sqlbuild/compiler/model_loop/__init__.py | 1 - src/sqlbuild/compiler/model_loop/constants.py | 15 -- .../compiler/model_loop/main/__init__.py | 1 - .../_build_native_declaration_contexts.py | 106 ---------- .../compiler/compile/_helpers/helpers.py | 4 +- 10 files changed, 215 insertions(+), 186 deletions(-) delete mode 100644 src/sqlbuild/compiler/model_loop/__init__.py delete mode 100644 src/sqlbuild/compiler/model_loop/constants.py delete mode 100644 src/sqlbuild/compiler/model_loop/main/__init__.py delete mode 100644 src/sqlbuild/compiler/model_loop/main/_build_native_declaration_contexts.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index 195988a36a..c61a7a7923 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyMapping, PyString}; +use pyo3::types::{PyDict, PyDictMethods, PyMapping, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_config::command_connection::constants::{ CLI_WAREHOUSE_SOURCE, CONNECTION_DEFAULT_WAREHOUSE_SOURCE, CONNECTION_WAREHOUSE_KEY, @@ -16,6 +16,8 @@ use sqlbuild_config::command_connection::main::unsupported_cli_warehouse::unsupp use sqlbuild_config::command_connection::models::AuthoredWarehouse; use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_model_config::templates::models::TemplateOptions; + use crate::bindings::_helpers::command_runtime::runtime_errors::{ expected_resolution_error, raised, }; @@ -23,6 +25,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::discovery::configuration::discover_project_configuration; +use crate::bindings::_helpers::model_config::template_data::expanded_template_data; const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; const SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; @@ -30,7 +33,6 @@ const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main._session_warehouse_support"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; -const EXPAND_TEMPLATE_MODULE: &str = "sqlbuild._native"; const RESOLVE_OPTIONS: (&[&str], &[&str]) = ( &[ "discovered_inputs", @@ -182,32 +184,17 @@ fn expand_authored( ("cli_vars", cli_vars.clone()), ], )?; - let expanded: Bound<'_, PyAny> = call_with( - &imported(py, EXPAND_TEMPLATE_MODULE, "expand_template_data")?, - &[ - ( - "value", - PyString::new(py, &found.authored.warehouse).into_any(), - ), - ("variables", runtime.get_item(1)?), - ("context_values", PyDict::new(py).into_any()), - ( - "context_label", - PyString::new(py, &found.authored.context_label()).into_any(), - ), - ( - "allow_context", - PyBool::new(py, false).to_owned().into_any(), - ), - ( - "preserve_context_tokens", - PyBool::new(py, false).to_owned().into_any(), - ), - ( - "preserve_unknown_context", - PyBool::new(py, false).to_owned().into_any(), - ), - ], + let expanded: Bound<'_, PyAny> = expanded_template_data( + PyString::new(py, &found.authored.warehouse).as_any(), + (runtime.get_item(1)?.downcast()?, &PyDict::new(py)), + ( + TemplateOptions { + allow_context: false, + preserve_context_tokens: false, + preserve_unknown_context: false, + }, + &found.authored.context_label(), + ), )?; let warehouse: String = expanded.str()?.extract()?; check_expanded_warehouse(&found.authored, &warehouse).map_err(|error| raised(py, error))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index 4d04157e08..ab1b1411f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -18,16 +18,18 @@ use sqlbuild_analysis::assembly::target_checks::main::target_error_message::targ use sqlbuild_analysis::assembly::target_checks::models::{LoaderCollision, MissingParts}; use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_model_config::templates::models::TemplateOptions; + use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; +use crate::bindings::_helpers::model_config::template_data::expanded_template_data; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; -const EXPAND_TEMPLATE_MODULE: &str = "sqlbuild._native"; const SETTING_HELP_MODULE: &str = "sqlbuild.errors.setting_help.main.setting_help"; /// Connection settings that identify the warehouse two targets write to. const CONNECTION_IDENTITY_KEYS: [&str; 7] = [ @@ -369,21 +371,17 @@ fn managed_source_namespace( let value: Bound<'_, PyDict> = PyDict::new(py); pyo3::types::PyDictMethods::set_item(&value, "database", database)?; pyo3::types::PyDictMethods::set_item(&value, "schema", schema)?; - let false_value: Bound<'_, PyAny> = pyo3::types::PyBool::new(py, false).to_owned().into_any(); - let expanded: Bound<'_, PyAny> = call_with( - &imported(py, EXPAND_TEMPLATE_MODULE, "expand_template_data")?, - &[ - ("value", value.into_any()), - ("variables", effective_vars.clone()), - ("context_values", PyDict::new(py).into_any()), - ( - "context_label", - PyString::new(py, LOADER_NAMESPACE_LABEL).into_any(), - ), - ("allow_context", false_value.clone()), - ("preserve_context_tokens", false_value.clone()), - ("preserve_unknown_context", false_value), - ], + let expanded: Bound<'_, PyAny> = expanded_template_data( + value.as_any(), + (effective_vars.downcast()?, &PyDict::new(py)), + ( + TemplateOptions { + allow_context: false, + preserve_context_tokens: false, + preserve_unknown_context: false, + }, + LOADER_NAMESPACE_LABEL, + ), )?; let part = |key: &str| -> PyResult> { let value: Bound<'_, PyAny> = expanded.get_item(key)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs index bc6133bf9b..facea17f1f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs @@ -3,8 +3,8 @@ use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyInt, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyInt, PyList, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; @@ -13,8 +13,13 @@ const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const SCOPE_LOOKUP_MODULE: &str = "sqlbuild.compiler.scopes.main.build_scope_lookup"; -const NATIVE_CONTEXTS_MODULE: &str = - "sqlbuild.compiler.model_loop.main._build_native_declaration_contexts"; +const NATIVE_MODULE: &str = "sqlbuild._native"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const BUILTINS_MODULE: &str = "builtins"; +const NO_DECLARATION_KIND: u8 = 255; +/// Declaration kinds by their native code: enum 0, constant 1, macro 2. +const DECLARATION_KINDS: [&str; 3] = ["ENUM", "CONSTANT", "MACRO"]; const SQL_VALUE_ERRORS_MODULE: &str = "sqlbuild.sql_values.exceptions"; const TYPES_MODULE: &str = "types"; const SCALAR_KINDS: [&str; 6] = ["string", "integer", "boolean", "float", "decimal", "null"]; @@ -142,17 +147,8 @@ pub(crate) fn declaration_scope_resolver<'py>( imported(py, TYPES_MODULE, "MappingProxyType")?.call1((&declarations,))?, )], )?; - let native_contexts: Bound<'py, PyAny> = call_with( - &imported( - py, - NATIVE_CONTEXTS_MODULE, - "build_native_declaration_contexts", - )?, - &[ - ("lookup", lookup.clone()), - ("declarations", declarations.into_any()), - ], - )?; + let native_contexts: Bound<'py, PyAny> = + native_declaration_contexts(&lookup, declarations.as_any())?; call_with( &imported(py, COMPILE_MODELS_MODULE, "DeclarationScopeResolver")?, &[ @@ -168,6 +164,173 @@ pub(crate) fn declaration_scope_resolver<'py>( ) } +/// The native code of declaration kind `kind`, or the no-kind code. +fn kind_code(kinds: &[Bound<'_, PyAny>], kind: &Bound<'_, PyAny>) -> u8 { + let mut code: u8 = 0; + for known in kinds { + if known.is(kind) { + return code; + } + code += 1; + } + NO_DECLARATION_KIND +} + +/// The native kind code of a runtime declaration value, by its class. +fn value_kind_code(classes: &[Bound<'_, PyAny>], value: &Bound<'_, PyAny>) -> PyResult { + let mut code: u8 = 0; + for class in classes { + if value.is_instance(class)? { + return Ok(code); + } + code += 1; + } + Ok(NO_DECLARATION_KIND) +} + +/// `{owner: list(items)}` of an index position mapping. +fn listed_positions<'py>(positions: &Bound<'py, PyAny>) -> PyResult> { + let listed: Bound<'py, PyDict> = PyDict::new(positions.py()); + for item in positions.call_method0("items")?.try_iter()? { + let (owner, items): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + listed.set_item( + owner, + PyList::new( + positions.py(), + items.try_iter()?.collect::>>()?, + )?, + )?; + } + Ok(listed) +} + +/// Hand the lookup's declaration positions and runtime values to the native classifier. +pub(crate) fn native_declaration_contexts<'py>( + lookup: &Bound<'py, PyAny>, + declarations: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = lookup.py(); + let kind_type: Bound<'py, PyAny> = imported(py, SCOPE_TYPES_MODULE, "DeclarationKind")?; + let kinds: Vec> = DECLARATION_KINDS + .iter() + .map(|kind| kind_type.getattr(*kind)) + .collect::>()?; + let value_classes: Vec> = vec![ + imported(py, DISCOVERY_MODELS_MODULE, "EnumDeclaration")?, + imported(py, DISCOVERY_MODELS_MODULE, "ConstantDeclaration")?, + imported(py, COMPILE_MODELS_MODULE, "LoadedMacro")?, + ]; + let index: Bound<'py, PyAny> = lookup.getattr("visibility_index")?; + let lookup_declarations: Bound<'py, PyAny> = lookup.getattr("declarations")?; + let identity_keys: Bound<'py, PyDict> = PyDict::new(py); + let mut positions: Vec> = Vec::new(); + for identity in index.getattr("identities")?.try_iter()? { + let identity: Bound<'py, PyAny> = identity?; + let key: Bound<'py, PyAny> = + identity_keys.call_method1("setdefault", (&identity, identity_keys.len()))?; + let value: Bound<'py, PyAny> = declarations.call_method1("get", (&identity,))?; + positions.push(PyTuple::new( + py, + [ + identity.clone(), + lookup_declarations.get_item(&identity)?.get_item(0)?, + identity.getattr("name")?, + kind_code(&kinds, &identity.getattr("kind")?) + .into_pyobject(py)? + .into_any(), + value.clone(), + value_kind_code(&value_classes, &value)? + .into_pyobject(py)? + .into_any(), + key, + ], + )?); + } + let reasons: Bound<'py, PyDict> = PyDict::new(py); + for reason in imported(py, SCOPE_TYPES_MODULE, "VisibilityReason")?.try_iter()? { + let reason: Bound<'py, PyAny> = reason?; + reasons.set_item(reason.getattr("value")?, reason)?; + } + let classes: Bound<'py, PyDict> = PyDict::new(py); + classes.set_item( + "visibility_record", + imported(py, SCOPE_MODELS_MODULE, "VisibilityRecord")?, + )?; + classes.set_item( + "resolution_context", + imported(py, COMPILE_MODELS_MODULE, "DeclarationResolutionContext")?, + )?; + classes.set_item( + "allocate", + imported(py, BUILTINS_MODULE, "object")?.getattr("__new__")?, + )?; + classes.set_item("reasons", reasons)?; + let keys: Bound<'py, PyDict> = PyDict::new(py); + keys.set_item("identity_keys", identity_keys)?; + keys.set_item("private_positions", index.getattr("private_positions")?)?; + keys.set_item("grants_by_resource", lookup.getattr("grants_by_resource")?)?; + let lookups: Bound<'py, PyDict> = PyDict::new(py); + lookups.set_item("resources_by_path", lookup.getattr("resources_by_path")?)?; + lookups.set_item("resources", lookup.getattr("resources")?)?; + lookups.set_item("declarations", &lookup_declarations)?; + lookups.set_item( + "index_declarations", + lookup.getattr("index")?.getattr("declarations")?, + )?; + for (name, class) in [ + ("resource_record", "ResourceRecord"), + ("ownership_root", "OwnershipRoot"), + ("resource_identity", "ResourceIdentity"), + ("declaration_identity", "DeclarationIdentity"), + ] { + lookups.set_item(name, imported(py, SCOPE_MODELS_MODULE, class)?)?; + } + lookups.set_item( + "resource_kind", + imported(py, SCOPE_TYPES_MODULE, "ResourceKind")?, + )?; + lookups.set_item("declaration_kind", kind_type)?; + lookups.set_item( + "private_scope", + imported(py, SCOPE_TYPES_MODULE, "ScopeKind")?.getattr("PRIVATE")?, + )?; + lookups.set_item( + "identity_error", + imported(py, SCOPE_ERRORS_MODULE, "InvalidQualifiedIdentityError")?, + )?; + lookups.set_item( + "path_error", + imported(py, SCOPE_ERRORS_MODULE, "InvalidScopePathError")?, + )?; + let global_positions: Bound<'py, PyList> = PyList::new( + py, + index + .getattr("global_positions")? + .try_iter()? + .collect::>>()?, + )?; + imported(py, NATIVE_MODULE, "NativeDeclarationContexts")?.call1(( + PyList::new(py, positions)?, + ( + global_positions, + listed_positions(&index.getattr("local_positions")?)?, + listed_positions(&index.getattr("inherited_positions")?)?, + ), + classes, + (keys, lookups), + )) +} + +/// Hand one scope lookup's positions and runtime values to the native classifier. +#[pyfunction] +#[pyo3(signature = (*, lookup, declarations))] +fn build_native_declaration_contexts<'py>( + lookup: &Bound<'py, PyAny>, + declarations: &Bound<'py, PyAny>, +) -> PyResult> { + native_declaration_contexts(lookup, declarations) +} + /// Canonical visibility for one authored path, projected onto runtime declaration values. pub(crate) fn resolved_declaration_context<'py>( resolver: &Bound<'py, PyAny>, @@ -308,6 +471,7 @@ fn resolve_declaration_context<'py>( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_native_declaration_contexts, module)?)?; module.add_function(wrap_pyfunction!(render_constant_declaration, module)?)?; module.add_function(wrap_pyfunction!(render_enum_member_value, module)?)?; module.add_function(wrap_pyfunction!(resolve_declaration_context, module)?)?; diff --git a/python-boundary.toml b/python-boundary.toml index d7f034c0e4..063255fa89 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 58336 +remaining_lines = 58213 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 318bf7c7da..eba0f173f4 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2771,6 +2771,11 @@ def fixture_projection_facts( sql: str, dialect: str | None, / ) -> tuple[list[tuple[str, str]], list[str], list[str]]: ... def find_macro_call_names(sql: str, /) -> tuple[str, ...]: ... +def build_native_declaration_contexts( + *, + lookup: ScopeLookup, + declarations: Mapping[DeclarationIdentity, EnumDeclaration | ConstantDeclaration | LoadedMacro], +) -> NativeDeclarationContexts: ... def render_constant_declaration( *, declaration: ConstantDeclaration, diff --git a/src/sqlbuild/compiler/model_loop/__init__.py b/src/sqlbuild/compiler/model_loop/__init__.py deleted file mode 100644 index 91f5295966..0000000000 --- a/src/sqlbuild/compiler/model_loop/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""The native model compile loop.""" diff --git a/src/sqlbuild/compiler/model_loop/constants.py b/src/sqlbuild/compiler/model_loop/constants.py deleted file mode 100644 index f1e2ec1996..0000000000 --- a/src/sqlbuild/compiler/model_loop/constants.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Codes the native model loop shares with its Rust bindings.""" - -from sqlbuild.compiler.scopes.types import DeclarationKind - -DECLARATION_KIND_CODES: dict[DeclarationKind, int] = { - DeclarationKind.ENUM: 0, - DeclarationKind.CONSTANT: 1, - DeclarationKind.MACRO: 2, -} -NO_DECLARATION_KIND: int = 255 -ENUM_REFERENCE_KIND_CODE: int = DECLARATION_KIND_CODES[DeclarationKind.ENUM] -UNCLOSED_QUOTE_STOP_CODE: int = 0 -UNCLOSED_BLOCK_COMMENT_STOP_CODE: int = 1 -INVALID_ENUM_REFERENCE_STOP_CODE: int = 2 -INVALID_CONSTANT_REFERENCE_STOP_CODE: int = 3 diff --git a/src/sqlbuild/compiler/model_loop/main/__init__.py b/src/sqlbuild/compiler/model_loop/main/__init__.py deleted file mode 100644 index 1956248370..0000000000 --- a/src/sqlbuild/compiler/model_loop/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native model loop entry points.""" diff --git a/src/sqlbuild/compiler/model_loop/main/_build_native_declaration_contexts.py b/src/sqlbuild/compiler/model_loop/main/_build_native_declaration_contexts.py deleted file mode 100644 index b77a6a61eb..0000000000 --- a/src/sqlbuild/compiler/model_loop/main/_build_native_declaration_contexts.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Hand one scope lookup to the native declaration context classifier.""" - -from __future__ import annotations - -from collections.abc import Mapping - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import ( - DeclarationResolutionContext, - LoadedMacro, -) -from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration -from sqlbuild.compiler.model_loop.constants import DECLARATION_KIND_CODES, NO_DECLARATION_KIND -from sqlbuild.compiler.scopes.exceptions import ( - InvalidQualifiedIdentityError, - InvalidScopePathError, -) -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationVisibilityIndex, - OwnershipRoot, - ResourceIdentity, - ResourceRecord, - ScopeLookup, - VisibilityRecord, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - ResourceKind, - ScopeKind, - VisibilityReason, -) - - -def build_native_declaration_contexts( - *, - lookup: ScopeLookup, - declarations: Mapping[DeclarationIdentity, EnumDeclaration | ConstantDeclaration | LoadedMacro], -) -> _native.NativeDeclarationContexts: - """Hand the lookup's declaration positions and runtime values to the native classifier.""" - - index: DeclarationVisibilityIndex = lookup.visibility_index - identity_keys: dict[DeclarationIdentity, int] = {} - positions: list[tuple[object, ...]] = [] - for identity in index.identities: - key: int = identity_keys.setdefault(identity, len(identity_keys)) - value: EnumDeclaration | ConstantDeclaration | LoadedMacro | None = declarations.get( - identity - ) - positions.append( - ( - identity, - lookup.declarations[identity][0], - identity.name, - DECLARATION_KIND_CODES.get(identity.kind, NO_DECLARATION_KIND), - value, - _value_kind(value), - key, - ) - ) - return _native.NativeDeclarationContexts( - positions, - ( - list(index.global_positions), - {owner: list(items) for owner, items in index.local_positions.items()}, - {owner: list(items) for owner, items in index.inherited_positions.items()}, - ), - { - "visibility_record": VisibilityRecord, - "resolution_context": DeclarationResolutionContext, - "allocate": object.__new__, - "reasons": {reason.value: reason for reason in VisibilityReason}, - }, - ( - { - "identity_keys": identity_keys, - "private_positions": index.private_positions, - "grants_by_resource": lookup.grants_by_resource, - }, - { - "resources_by_path": lookup.resources_by_path, - "resources": lookup.resources, - "declarations": lookup.declarations, - "index_declarations": lookup.index.declarations, - "resource_record": ResourceRecord, - "ownership_root": OwnershipRoot, - "resource_identity": ResourceIdentity, - "declaration_identity": DeclarationIdentity, - "resource_kind": ResourceKind, - "declaration_kind": DeclarationKind, - "private_scope": ScopeKind.PRIVATE, - "identity_error": InvalidQualifiedIdentityError, - "path_error": InvalidScopePathError, - }, - ), - ) - - -def _value_kind(value: EnumDeclaration | ConstantDeclaration | LoadedMacro | None) -> int: - if isinstance(value, EnumDeclaration): - return DECLARATION_KIND_CODES[DeclarationKind.ENUM] - if isinstance(value, ConstantDeclaration): - return DECLARATION_KIND_CODES[DeclarationKind.CONSTANT] - if isinstance(value, LoadedMacro): - return DECLARATION_KIND_CODES[DeclarationKind.MACRO] - return NO_DECLARATION_KIND diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 8bae1a3e59..d01b4e432b 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( assemble_project, build_compile_inputs, + build_native_declaration_contexts, discover_project_inputs, expand_sql_macros, infer_native_query_columns, @@ -60,9 +61,6 @@ InferredNullability, ) from sqlbuild.compiler.manifest.main.build import build_manifest -from sqlbuild.compiler.model_loop.main._build_native_declaration_contexts import ( - build_native_declaration_contexts, -) from sqlbuild.compiler.planner._helpers.graph.core import build_execution_upstream_deps from sqlbuild.compiler.planner._helpers.resolve.refs import resolve_ref_references from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup From e4e24397e865cf064379f107c3876c0a7907ceb5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 04:35:16 +0100 Subject: [PATCH 16/40] refactor: split selectors and compile static command contexts natively Unit-test selector splitting, configured-rule and analysis model selection, and the static command compile/scope/relation phases are native bindings; Rust callers use sqlbuild._native. --- .../_helpers/audit_command/command.rs | 2 +- .../bindings/_helpers/check/check_command.rs | 2 +- .../bindings/_helpers/compile_driver/mod.rs | 1 + .../_helpers/compile_driver/pipeline.rs | 2 +- .../_helpers/compile_driver/project.rs | 2 +- .../_helpers/compile_driver/registration.rs | 3 +- .../compile_driver/selection_phases.rs | 329 ++++++++++++++++++ .../bindings/_helpers/seed_command/command.rs | 2 +- .../_helpers/sql_test_command/test_command.rs | 2 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 24 ++ .../pipeline/_helpers/analysis_selection.py | 104 ------ .../compiler/pipeline/main/_static_command.py | 96 ----- .../main/test_compile_phase_timings.py | 5 +- 14 files changed, 364 insertions(+), 212 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs delete mode 100644 src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py delete mode 100644 src/sqlbuild/compiler/pipeline/main/_static_command.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index 1489665b2c..a438a7c984 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -17,7 +17,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; -const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main._static_command"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const AUDIT_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._audit"; const COST_CONTEXT_MODULE: &str = "sqlbuild.cost.classes.cost_context"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 6d6a88e47f..4ca5e4664f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -25,7 +25,7 @@ const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_ const CHECK_CORE_MODULE: &str = "sqlbuild.cli.commands._helpers.check.core"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main._static_command"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const RELATION_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._relation_plan"; const SELECTOR_PARSE_MODULE: &str = "sqlbuild.compiler.planner.main.selection.selector_parse"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index 7ebf3fd7ae..bbda2f8952 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -19,5 +19,6 @@ pub(crate) mod python_calls; pub(crate) mod python_plan_entries; pub(crate) mod registration; pub(crate) mod relation_targets; +pub(crate) mod selection_phases; pub(crate) mod target_defaults; pub(crate) mod targets; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index 1c8c6a3be5..70bee37f71 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -17,7 +17,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.analysis_selection"; +const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const TIMING_TRACKER_MODULE: &str = "sqlbuild.diagnostics.classes.build_phase_timing_tracker"; const RULE_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 2f8728e766..103aac491a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -26,7 +26,7 @@ const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; const PROMOTION_MODULE: &str = "sqlbuild._native"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; -const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.analysis_selection"; +const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._record"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const COLLECTION_RENDERING_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs index 81700b0724..050f838115 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::compile_driver::{ analysis, assembly, command, deferral, diff, discovery, input_reads, inputs, pipeline, - pipeline_entries, project, python_plan_entries, relation_targets, targets, + pipeline_entries, project, python_plan_entries, relation_targets, selection_phases, targets, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -21,5 +21,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff::register(module)?; python_plan_entries::register(module)?; relation_targets::register(module)?; + selection_phases::register(module)?; targets::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs new file mode 100644 index 0000000000..29e2b31dd0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -0,0 +1,329 @@ +//! Unit-test selector splitting, analysis and rule selection, and static command compilation. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; +const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const SPLIT_SELECTORS_MODULE: &str = + "sqlbuild.compiler.planner.main.selection._split_unit_test_selectors"; +const PLANNER_SCOPES_MODULE: &str = + "sqlbuild.compiler.planner.main.selection._resolve_planner_scopes"; +const STATIC_SCOPE_MODULE: &str = "sqlbuild.compiler.planner.main.commands._scope"; +const STATIC_RELATIONS_MODULE: &str = "sqlbuild.compiler.planner.main.commands._relations"; +const DATACLASSES_MODULE: &str = "dataclasses"; + +/// Whether a selection names nothing to select or exclude. +fn unselective(selection: &Bound<'_, PyAny>) -> PyResult { + Ok(!selection.getattr("select")?.is_truthy()? && !selection.getattr("exclude")?.is_truthy()?) +} + +/// Options whose selectors name only resources, with unit tests selected apart. +fn unit_test_selectors_applied<'py>( + options: &Bound<'py, PyAny>, + discovered: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let resolved: Bound<'py, PyAny> = if options.is_none() { + imported(py, PIPELINE_MODELS_MODULE, "CompilePipelineOptions")?.call0()? + } else { + options.clone() + }; + if unselective(&resolved)? { + return Ok(resolved); + } + let split: Bound<'py, PyAny> = call_with( + &imported( + py, + SPLIT_SELECTORS_MODULE, + "split_project_unit_test_selectors", + )?, + &[ + ("select", resolved.getattr("select")?), + ("exclude", resolved.getattr("exclude")?), + ("discovered_inputs", discovered.clone()), + ( + "accepts_unit_tests", + resolved.getattr("accepts_unit_test_selectors")?, + ), + ], + )?; + imported(py, DATACLASSES_MODULE, "replace")?.call( + (resolved,), + Some(&keywords( + py, + &[ + ("select", split.getattr("select")?), + ("exclude", split.getattr("exclude")?), + ("sql_test_selection", split.getattr("sql_test_selection")?), + ], + )?), + ) +} + +/// The planner scopes `selection` resolves over `project`. +fn planner_scopes<'py>( + project: &Bound<'py, PyAny>, + selection: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + call_with( + &imported(py, PLANNER_SCOPES_MODULE, "resolve_planner_scopes")?, + &[ + ("project", project.clone()), + ( + "selection", + call_with( + &imported(py, PLANNER_MODELS_MODULE, "PlannerSelection")?, + &[ + ("select", selection.getattr("select")?), + ("exclude", selection.getattr("exclude")?), + ], + )?, + ), + ( + "policies", + call_with( + &imported(py, PLANNER_MODELS_MODULE, "PlannerPolicies")?, + &[("auto_load_sources", selection.getattr("auto_load_sources")?)], + )?, + ), + ], + ) +} + +/// The stale-warning scope's models of `scopes`. +fn scoped_models<'py>(scopes: &Bound<'py, PyAny>) -> PyResult> { + scopes + .getattr("stale_warning_scope")? + .getattr("models_by_name") +} + +/// Return options whose selectors name only resources, with unit tests selected apart. +#[pyfunction] +#[pyo3(signature = (*, options, discovered_inputs))] +fn apply_unit_test_selectors<'py>( + options: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + unit_test_selectors_applied(options, discovered_inputs) +} + +/// Align configured-rule subjects with models that received deep SQL analysis. +#[pyfunction] +#[pyo3(signature = (*, options, discovered_inputs, project))] +fn resolve_configured_rule_selection<'py>( + options: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = options.py(); + if !options + .getattr("sql_test_selection")? + .getattr("models_selected")? + .is_truthy()? + { + return Ok(PyFrozenSet::empty(py)?.into_any()); + } + let selection: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_compile_analysis_selection")?, + &[ + ("options", options.clone()), + ("discovered_inputs", discovered_inputs.clone()), + ], + )?; + if selection.is_none() || unselective(&selection)? { + return Ok(py.None().into_bound(py)); + } + let models: Bound<'py, PyAny> = scoped_models(&planner_scopes(project, &selection)?)?; + let keys: Vec> = models + .call_method0("values")? + .try_iter()? + .map(|model| model?.getattr("key")) + .collect::>()?; + Ok(PyFrozenSet::new(py, keys)?.into_any()) +} + +/// Return the models selected for deep analysis, or None to analyse every model. +#[pyfunction] +#[pyo3(signature = (*, compile_inputs, inference_profile, selection))] +fn resolve_analysis_model_names<'py>( + compile_inputs: &Bound<'py, PyAny>, + inference_profile: &Bound<'py, PyAny>, + selection: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = compile_inputs.py(); + if selection.is_none() || unselective(selection)? { + return Ok(py.None().into_bound(py)); + } + let structural_project: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "assemble_project")?, + &[ + ("inputs", compile_inputs.clone()), + ("inference_profile", inference_profile.clone()), + ( + "skip_column_inference", + pyo3::types::PyBool::new(py, true).to_owned().into_any(), + ), + ("analysis_model_names", PyFrozenSet::empty(py)?.into_any()), + ], + )?; + let models: Bound<'py, PyAny> = + scoped_models(&planner_scopes(&structural_project, selection)?)?; + Ok(PyFrozenSet::new(py, models.try_iter()?.collect::>>()?)?.into_any()) +} + +/// Locations of the deferred target's relations, when the options defer to one. +fn deferred_locations<'py>( + (discovered, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + options: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let defer_to: Bound<'py, PyAny> = options.getattr("defer_to")?; + if defer_to.is_none() { + return Ok(py.None().into_bound(py)); + } + let deferred_target: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_deferred_target_config")?, + &[ + ("discovered_inputs", discovered.clone()), + ("defer_to", defer_to), + ( + "current_target_name", + project.getattr("effective_target_name")?, + ), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "build_deferred_locations")?, + &[ + ("project", project.clone()), + ("deferred_target_config", deferred_target), + ("effective_vars", project.getattr("effective_vars")?), + ("default_schema", adapter.call_method0("default_schema")?), + ( + "default_database", + adapter.call_method0("default_database")?, + ), + ( + "render_qualified_name", + adapter.getattr("render_qualified_name")?, + ), + ], + ) +} + +/// Run canonical compile, selector, and relation phases without build planning. +#[pyfunction] +#[pyo3(signature = (**request))] +fn compile_static_command_context<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request: Request<'_, 'py> = Request::new("compile_static_command_context", request)?; + let py: Python<'py> = request.py(); + let none = || py.None().into_bound(py); + let discovered: Bound<'py, PyAny> = request.get("discovered_inputs")?; + let adapter: Bound<'py, PyAny> = request.get("adapter")?; + let options: Bound<'py, PyAny> = + unit_test_selectors_applied(&request.get("options")?, &discovered)?; + let mut selected_keys: Bound<'py, PyAny> = + request.optional("selected_keys")?.unwrap_or_else(none); + if selected_keys.is_none() + && !options + .getattr("sql_test_selection")? + .getattr("models_selected")? + .is_truthy()? + { + selected_keys = PyFrozenSet::empty(py)?.into_any(); + } + let compiled: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "compile_project_phase")?, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ("options", options.clone()), + ( + "on_progress", + request.optional("on_progress")?.unwrap_or_else(none), + ), + ], + )?; + let project: Bound<'py, PyAny> = compiled.getattr("project")?; + let deferred: Bound<'py, PyAny> = + deferred_locations((&discovered, &adapter), &options, &project)?; + let scope: Bound<'py, PyAny> = call_with( + &imported(py, STATIC_SCOPE_MODULE, "resolve_static_command_scope")?, + &[ + ("project", project.clone()), + ( + "selection", + call_with( + &imported(py, PLANNER_MODELS_MODULE, "PlannerSelection")?, + &[ + ("select", options.getattr("select")?), + ("exclude", options.getattr("exclude")?), + ("selected_keys", selected_keys), + ("sql_test_selection", options.getattr("sql_test_selection")?), + ], + )?, + ), + ("auto_load_sources", options.getattr("auto_load_sources")?), + ], + )?; + let relations: Bound<'py, PyAny> = call_with( + &imported( + py, + STATIC_RELATIONS_MODULE, + "resolve_static_relation_context", + )?, + &[ + ("project", project.clone()), + ("adapter", adapter), + ("scope", scope.clone()), + ( + "deferral", + call_with( + &imported(py, PLANNER_MODELS_MODULE, "DeferralInputs")?, + &[ + ("deferred_locations", deferred), + ("defer_sources_to", options.getattr("defer_sources_to")?), + ( + "source_deferral_enabled", + options.getattr("source_deferral_enabled")?, + ), + ], + )?, + ), + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ( + "relation_keys", + request.optional("relation_keys")?.unwrap_or_else(none), + ), + ], + )?; + call_with( + &imported(py, PIPELINE_MODELS_MODULE, "StaticCommandContext")?, + &[ + ("project", project), + ("scope", scope), + ("relations", relations), + ("connection_config", compiled.getattr("connection_config")?), + ("compile_seconds", compiled.getattr("compile_seconds")?), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(apply_unit_test_selectors, module)?)?; + module.add_function(wrap_pyfunction!(resolve_configured_rule_selection, module)?)?; + module.add_function(wrap_pyfunction!(resolve_analysis_model_names, module)?)?; + module.add_function(wrap_pyfunction!(compile_static_command_context, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 61e4d53156..02231a9701 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -20,7 +20,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; -const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main._static_command"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const SEED_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._seed"; const SEED_STATE_MODULE: &str = "sqlbuild.compiler.planner.main.commands._seed_state"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 5c20ac5e8e..5ecaaaa904 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -32,7 +32,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; -const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main._static_command"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const TEST_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._sql_test"; const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; diff --git a/python-boundary.toml b/python-boundary.toml index 063255fa89..b91c06bd6b 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 58213 +remaining_lines = 58013 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index eba0f173f4..5c094dc9cd 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2514,6 +2514,30 @@ def resolve_effective_scenario_config( def scenario_local_type_overrides_for_dialect( *, scenario_config: ScenarioConfig, sql_analysis_dialect: str | None ) -> dict[str, str]: ... +def apply_unit_test_selectors( + *, options: CompilePipelineOptions | None, discovered_inputs: DiscoveredProjectInputs +) -> CompilePipelineOptions: ... +def resolve_configured_rule_selection( + *, + options: CompilePipelineOptions, + discovered_inputs: DiscoveredProjectInputs, + project: CompiledProjectView, +) -> frozenset[CompiledObjectKey] | None: ... +def resolve_analysis_model_names( + *, + compile_inputs: CompileInputsView, + inference_profile: ExpressionInferenceProfile, + selection: CompileAnalysisSelection | None, +) -> frozenset[str] | None: ... +def compile_static_command_context( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + options: CompilePipelineOptions, + selected_keys: frozenset[CompiledObjectKey] | None = None, + relation_keys: frozenset[CompiledObjectKey] | None = None, + on_progress: Callable[[str], None] | None = None, +) -> StaticCommandContext: ... def run_clone_pipeline_with_options( *, discovered_inputs: DiscoveredProjectInputs, diff --git a/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py b/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py deleted file mode 100644 index 596615324a..0000000000 --- a/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Selection policies applied to compile pipeline options.""" - -from __future__ import annotations - -from dataclasses import replace - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import ( - CompileAnalysisSelection, - CompiledObjectKey, -) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineOptions -from sqlbuild.compiler.planner.main.selection._resolve_planner_scopes import ( - resolve_planner_scopes, -) -from sqlbuild.compiler.planner.main.selection._split_unit_test_selectors import ( - split_project_unit_test_selectors, -) -from sqlbuild.compiler.planner.models import ( - PlannerPolicies, - PlannerScopeResolution, - PlannerSelection, - UnitTestSelectorSplit, -) - - -def apply_unit_test_selectors( - *, options: CompilePipelineOptions | None, discovered_inputs: DiscoveredProjectInputs -) -> CompilePipelineOptions: - """Return options whose selectors name only resources, with unit tests selected apart.""" - - resolved: CompilePipelineOptions = options if options is not None else CompilePipelineOptions() - if not resolved.select and not resolved.exclude: - return resolved - split: UnitTestSelectorSplit = split_project_unit_test_selectors( - select=resolved.select, - exclude=resolved.exclude, - discovered_inputs=discovered_inputs, - accepts_unit_tests=resolved.accepts_unit_test_selectors, - ) - return replace( - resolved, - select=split.select, - exclude=split.exclude, - sql_test_selection=split.sql_test_selection, - ) - - -def resolve_configured_rule_selection( - *, - options: CompilePipelineOptions, - discovered_inputs: DiscoveredProjectInputs, - project: CompiledProjectView, -) -> frozenset[CompiledObjectKey] | None: - """Align configured-rule subjects with models that received deep SQL analysis.""" - - if not options.sql_test_selection.models_selected: - return frozenset() - analysis_selection: CompileAnalysisSelection | None = ( - _native.resolve_compile_analysis_selection( - options=options, - discovered_inputs=discovered_inputs, - ) - ) - if analysis_selection is None or ( - not analysis_selection.select and not analysis_selection.exclude - ): - return None - scopes: PlannerScopeResolution = resolve_planner_scopes( - project=project, - selection=PlannerSelection( - select=analysis_selection.select, - exclude=analysis_selection.exclude, - ), - policies=PlannerPolicies(auto_load_sources=analysis_selection.auto_load_sources), - ) - return frozenset(model.key for model in scopes.stale_warning_scope.models_by_name.values()) - - -def resolve_analysis_model_names( - *, - compile_inputs: CompileInputsView, - inference_profile: ExpressionInferenceProfile, - selection: CompileAnalysisSelection | None, -) -> frozenset[str] | None: - """Return the models selected for deep analysis, or None to analyse every model.""" - - if selection is None or (not selection.select and not selection.exclude): - return None - structural_project: CompiledProjectView = _native.assemble_project( - inputs=compile_inputs, - inference_profile=inference_profile, - skip_column_inference=True, - analysis_model_names=frozenset(), - ) - scopes: PlannerScopeResolution = resolve_planner_scopes( - project=structural_project, - selection=PlannerSelection(select=selection.select, exclude=selection.exclude), - policies=PlannerPolicies(auto_load_sources=selection.auto_load_sources), - ) - return frozenset(scopes.stale_warning_scope.models_by_name) diff --git a/src/sqlbuild/compiler/pipeline/main/_static_command.py b/src/sqlbuild/compiler/pipeline/main/_static_command.py deleted file mode 100644 index b6bea2c6d4..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/_static_command.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Public static command compilation entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild._native import ( - build_deferred_locations, - compile_project_phase, - resolve_deferred_target_config, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.analysis_selection import apply_unit_test_selectors -from sqlbuild.compiler.pipeline.models import ( - CompiledProjectPhaseResult, - CompilePipelineOptions, - StaticCommandContext, -) -from sqlbuild.compiler.planner.main.commands._relations import resolve_static_relation_context -from sqlbuild.compiler.planner.main.commands._scope import resolve_static_command_scope -from sqlbuild.compiler.planner.models import ( - DeferralInputs, - PlannerRelationsContext, - PlannerScope, - PlannerSelection, -) -from sqlbuild.spec.contracts.models import TargetConfig - - -def compile_static_command_context( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - options: CompilePipelineOptions, - selected_keys: frozenset[CompiledObjectKey] | None = None, - relation_keys: frozenset[CompiledObjectKey] | None = None, - on_progress: Callable[[str], None] | None = None, -) -> StaticCommandContext: - """Run canonical compile, selector, and relation phases without build planning.""" - - options = apply_unit_test_selectors(options=options, discovered_inputs=discovered_inputs) - if selected_keys is None and not options.sql_test_selection.models_selected: - selected_keys = frozenset() - compiled: CompiledProjectPhaseResult = compile_project_phase( - discovered_inputs=discovered_inputs, - adapter=adapter, - options=options, - on_progress=on_progress, - ) - deferred_locations: dict[str, CompiledRelationLocation] | None = None - if options.defer_to is not None: - deferred_target: TargetConfig = resolve_deferred_target_config( - discovered_inputs=discovered_inputs, - defer_to=options.defer_to, - current_target_name=compiled.project.effective_target_name, - ) - deferred_locations = build_deferred_locations( - project=compiled.project, - deferred_target_config=deferred_target, - effective_vars=compiled.project.effective_vars, - default_schema=adapter.default_schema(), - default_database=adapter.default_database(), - render_qualified_name=adapter.render_qualified_name, - ) - scope: PlannerScope = resolve_static_command_scope( - project=compiled.project, - selection=PlannerSelection( - select=options.select, - exclude=options.exclude, - selected_keys=selected_keys, - sql_test_selection=options.sql_test_selection, - ), - auto_load_sources=options.auto_load_sources, - ) - relations: PlannerRelationsContext = resolve_static_relation_context( - project=compiled.project, - adapter=adapter, - scope=scope, - deferral=DeferralInputs( - deferred_locations=deferred_locations, - defer_sources_to=options.defer_sources_to, - source_deferral_enabled=options.source_deferral_enabled, - ), - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - relation_keys=relation_keys, - ) - return StaticCommandContext( - project=compiled.project, - scope=scope, - relations=relations, - connection_config=compiled.connection_config, - compile_seconds=compiled.compile_seconds, - ) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index a3594a3e41..c20ea521fa 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -10,7 +10,6 @@ import sqlbuild._native as _native from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline._helpers import analysis_selection from sqlbuild.compiler.pipeline.main import _plan_result as plan_result from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput @@ -68,9 +67,7 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di monkeypatch.setattr(open_connection, "open_connection_with_hooks", Mock(return_value=object())) monkeypatch.setattr(plan_result, "build_pipeline_result", Mock(return_value=pipeline_result)) monkeypatch.setattr(load_config, "load_rules_config", Mock(return_value=RulesConfig())) - monkeypatch.setattr( - analysis_selection, "resolve_configured_rule_selection", Mock(return_value=None) - ) + monkeypatch.setattr(_native, "resolve_configured_rule_selection", Mock(return_value=None)) monkeypatch.setattr( run_rules, "run_rules", From bc5379c921b12943b61e451c59a8f7a36dd94240 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 04:37:10 +0100 Subject: [PATCH 17/40] style: count declaration kind codes with a zipped range --- .../src/bindings/_helpers/render/declaration_values.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs index facea17f1f..3cd5fa6bd4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs @@ -166,24 +166,20 @@ pub(crate) fn declaration_scope_resolver<'py>( /// The native code of declaration kind `kind`, or the no-kind code. fn kind_code(kinds: &[Bound<'_, PyAny>], kind: &Bound<'_, PyAny>) -> u8 { - let mut code: u8 = 0; - for known in kinds { + for (code, known) in (0_u8..).zip(kinds.iter()) { if known.is(kind) { return code; } - code += 1; } NO_DECLARATION_KIND } /// The native kind code of a runtime declaration value, by its class. fn value_kind_code(classes: &[Bound<'_, PyAny>], value: &Bound<'_, PyAny>) -> PyResult { - let mut code: u8 = 0; - for class in classes { + for (code, class) in (0_u8..).zip(classes.iter()) { if value.is_instance(class)? { return Ok(code); } - code += 1; } Ok(NO_DECLARATION_KIND) } From 0b2b06048b4b25d938ef3be1e2e4b409e70b35b7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 04:25:34 +0100 Subject: [PATCH 18/40] refactor: port the transient status reporter to native --- .../main/format_status_spinner_line.rs | 27 ++ crates/sqlbuild-core/src/cli_text/main/mod.rs | 1 + .../src/cli_text/tests/test_cli_text.rs | 44 ++- .../src/cli_text/tests/test_types.rs | 8 + .../bindings/_helpers/build_progress/mod.rs | 1 + .../_helpers/build_progress/registration.rs | 7 +- .../build_progress/transient_status.rs | 296 ++++++++++++++++++ .../_helpers/cli/progress_reporters.rs | 11 +- .../_helpers/compile_driver/command.rs | 13 +- .../_helpers/contract_command/command.rs | 14 +- .../bindings/_helpers/dbt_command/support.rs | 13 +- .../_helpers/format_command/command.rs | 11 +- .../_helpers/refactor/refactor_command.rs | 8 +- .../_helpers/scenario_command/presentation.rs | 11 +- .../_helpers/sql_test_command/test_command.rs | 7 +- src/sqlbuild/_native.pyi | 14 +- .../_helpers/janitor_runtime/planning.py | 2 +- .../classes/dbt_init_progress_reporter.py | 2 +- .../dbt/_helpers/pipeline/execute.py | 2 +- .../dbt/_helpers/runtime/event_stream.py | 2 +- .../classes/transient_status_reporter.py | 139 -------- .../presentation/main/_maybe_status.py | 2 +- .../cli/commands/main/compile/helpers.py | 8 + .../main/compile/test_compile_command.py | 50 +-- .../integrations/dbt/test_event_stream.py | 28 +- .../presentation/classes/test_status.py | 7 +- 26 files changed, 446 insertions(+), 282 deletions(-) create mode 100644 crates/sqlbuild-core/src/cli_text/main/format_status_spinner_line.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs delete mode 100644 src/sqlbuild/presentation/classes/transient_status_reporter.py diff --git a/crates/sqlbuild-core/src/cli_text/main/format_status_spinner_line.rs b/crates/sqlbuild-core/src/cli_text/main/format_status_spinner_line.rs new file mode 100644 index 0000000000..8dea6ccb41 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_status_spinner_line.rs @@ -0,0 +1,27 @@ +//! The live line of a transient status: a spinner frame and the current message. + +use crate::cli_text::constants::ELLIPSIS; +use crate::cli_text::models::CliStyle; + +/// The frame and the first message line, cut to fit `columns` (`0` leaves it uncut). +#[must_use] +pub fn format_status_spinner_line( + frame: &str, + message: &str, + columns: usize, + style: CliStyle, +) -> String { + let text: &str = message.lines().next().unwrap_or(""); + let width: usize = columns.saturating_sub(frame.chars().count() + 2); + let fitted: String = if columns == 0 || text.chars().count() <= width { + text.to_owned() + } else { + format!( + "{}{ELLIPSIS}", + text.chars() + .take(width.saturating_sub(ELLIPSIS.len())) + .collect::() + ) + }; + format!("{} {fitted}", style.success(frame)) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/mod.rs b/crates/sqlbuild-core/src/cli_text/main/mod.rs index 60ec06a502..b6aced55d8 100644 --- a/crates/sqlbuild-core/src/cli_text/main/mod.rs +++ b/crates/sqlbuild-core/src/cli_text/main/mod.rs @@ -12,6 +12,7 @@ pub mod format_phase_line; pub mod format_plan_completion; pub mod format_plan_text; pub mod format_status_cell; +pub mod format_status_spinner_line; pub mod format_summary_footer; pub mod format_surface_header; pub mod format_tree_connector; diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index ddac7563c2..ff0596afc9 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -3,6 +3,7 @@ use crate::cli_text::main::format_compile_text::format_compile_text; use crate::cli_text::main::format_dbt_interop_plan::format_dbt_interop_plan; use crate::cli_text::main::format_plan_completion::format_plan_completion; use crate::cli_text::main::format_plan_text::format_plan_text; +use crate::cli_text::main::format_status_spinner_line::format_status_spinner_line; use crate::cli_text::main::semantic_coverage_notice::semantic_coverage_notice; use crate::cli_text::models::{ CliStyle, CompileTextReport, DbtInteropPlanText, DbtLsNodeText, DiagnosticText, DisplayLimit, @@ -11,7 +12,7 @@ use crate::cli_text::models::{ use crate::cli_text::tests::helpers::{diagnostic, location, plan_report, table_model}; use crate::cli_text::tests::test_types::{ CompileTextTestCase, DbtPlanTextTestCase, DiagnosticsTextTestCase, NoticeTestCase, - PlanTextTestCase, + PlanTextTestCase, StatusSpinnerLineTestCase, }; /// Expectations recorded from the retired Python formatter. @@ -232,3 +233,44 @@ fn given_dbt_interop_plan_when_formatting_then_matches_python_text() { ); } } + +#[test] +fn given_status_message_when_rendering_spinner_line_then_fits_terminal() { + let test_cases = [ + StatusSpinnerLineTestCase { + description: "short message is kept with a green frame", + message: "Compiling project graph...", + columns: 80, + style: CliStyle { use_color: true }, + expected_line: "\x1b[32m\u{280b}\x1b[0m Compiling project graph...", + }, + StatusSpinnerLineTestCase { + description: "long message is cut below the terminal width", + message: "Compiling project graph...", + columns: 15, + style: CliStyle { use_color: false }, + expected_line: "\u{280b} Compiling...", + }, + StatusSpinnerLineTestCase { + description: "unknown width keeps only the first line", + message: "first line\nsecond line", + columns: 0, + style: CliStyle { use_color: false }, + expected_line: "\u{280b} first line", + }, + ]; + + for test_case in test_cases { + assert_eq!( + format_status_spinner_line( + "\u{280b}", + test_case.message, + test_case.columns, + test_case.style + ), + test_case.expected_line, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_types.rs b/crates/sqlbuild-core/src/cli_text/tests/test_types.rs index ed807492b8..e733365cb3 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_types.rs @@ -32,3 +32,11 @@ pub(super) struct DbtPlanTextTestCase { pub(super) max_entries: Option, pub(super) expected_text: &'static str, } + +pub(super) struct StatusSpinnerLineTestCase { + pub(super) description: &'static str, + pub(super) message: &'static str, + pub(super) columns: usize, + pub(super) style: CliStyle, + pub(super) expected_line: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs index 2785c8691b..e0efa09893 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs @@ -6,3 +6,4 @@ pub(crate) mod plan_facts; pub(crate) mod registration; pub(crate) mod result_text; pub(crate) mod terminal; +pub(crate) mod transient_status; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs index 42ec0ba298..591e3c48e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs @@ -1,14 +1,15 @@ -//! Register the native build, audit and test progress reporters and the build footer. +//! Register the native build, audit and test progress reporters, the build footer and the transient status. use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::build_progress::{ - audit_progress, build_callbacks, footer, nested_progress, + audit_progress, build_callbacks, footer, nested_progress, transient_status, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { audit_progress::register(module)?; build_callbacks::register(module)?; footer::register(module)?; - nested_progress::register(module) + nested_progress::register(module)?; + transient_status::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs new file mode 100644 index 0000000000..de45f63a02 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs @@ -0,0 +1,296 @@ +//! One transient status line with a spinner on a terminal and plain lines elsewhere. + +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::cli_text::main::format_status_spinner_line::format_status_spinner_line; +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::bindings::_helpers::build_progress::terminal::{ + CLEAR_LINE, HIDE_CURSOR, SHOW_CURSOR, SPINNER_FRAMES, SPINNER_TICK_SECONDS, claim, flush, + release, shared_lines, start_spinner, terminal_columns, tty_flag, write, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{imported, within}; + +const SPINNERS_DISABLED_MODULE: &str = "sqlbuild.presentation.main._progress_spinners_disabled"; +const PREFLIGHT_COMPLETION_PREFIX: &str = "Prepared "; + +/// Mutable reporter state; only held briefly and never across Python calls. +#[derive(Default)] +struct StatusState { + message: Option, + active: bool, + drawn: bool, + frame: usize, + stop_event: Option>, + exit_hook_registered: bool, +} + +/// Render one transient status line with optional spinner updates. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct TransientStatusReporter { + stream: Py, + lines: Py, + lock: Py, + style: CliStyle, + enabled: bool, + state: Mutex, +} + +impl TransientStatusReporter { + fn state(&self) -> MutexGuard<'_, StatusState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn locked(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { + within(self.lock.bind(py), |_| body()) + } + + fn write_persistent(&self, py: Python<'_>, text: &str) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("stream", self.stream.bind(py))?; + options.set_item("text", text)?; + self.lines + .bind(py) + .call_method("write_persistent", (), Some(&options))?; + Ok(()) + } + + fn write_message(&self, py: Python<'_>, message: &str, dim_output: bool) -> PyResult<()> { + let formatted: String = if dim_output { + self.style.muted(message) + } else { + message.to_owned() + }; + self.write_persistent(py, &format!("{formatted}\n")) + } + + fn draw(&self, py: Python<'_>) -> PyResult<()> { + let (message, frame) = { + let mut state = self.state(); + let Some(message) = state.message.clone().filter(|_| state.drawn) else { + return Ok(()); + }; + let frame: usize = state.frame; + state.frame = (frame + 1) % SPINNER_FRAMES.len(); + (message, frame) + }; + let line: String = format_status_spinner_line( + SPINNER_FRAMES[frame], + &message, + terminal_columns(py)?, + self.style, + ); + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, &format!("{CLEAR_LINE}{line}"))?; + flush(stream) + } + + fn enter_status(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + let register_hook: bool = + !std::mem::replace(&mut reporter.state().exit_hook_registered, true); + if register_hook { + imported(py, "atexit", "register")?.call1((slf.getattr("close")?,))?; + } + let stream: &Bound<'_, PyAny> = reporter.stream.bind(py); + write(stream, HIDE_CURSOR)?; + reporter.state().drawn = true; + reporter.draw(py)?; + let (event, _thread) = start_spinner(slf.as_any())?; + reporter.state().stop_event = Some(event.unbind()); + Ok(()) + } + + fn exit_status(&self, py: Python<'_>) -> PyResult<()> { + let (drawn, event) = { + let mut state = self.state(); + (std::mem::take(&mut state.drawn), state.stop_event.take()) + }; + if let Some(event) = event { + event.bind(py).call_method0("set")?; + } + if drawn { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, &format!("{CLEAR_LINE}{SHOW_CURSOR}"))?; + flush(stream)?; + } + Ok(()) + } +} + +#[pymethods] +impl TransientStatusReporter { + #[new] + #[pyo3(signature = (*, stream, use_color = false, enabled = true))] + fn new( + py: Python<'_>, + stream: &Bound<'_, PyAny>, + use_color: bool, + enabled: bool, + ) -> PyResult { + let lines: Bound<'_, PyAny> = shared_lines(py)?; + let spinners_disabled: bool = + imported(py, SPINNERS_DISABLED_MODULE, "progress_spinners_disabled")? + .call0()? + .is_truthy()?; + Ok(Self { + enabled: enabled && !spinners_disabled && tty_flag(stream)?, + stream: stream.clone().unbind(), + lock: lines.getattr("lock")?.unbind(), + lines: lines.unbind(), + style: CliStyle { use_color }, + state: Mutex::new(StatusState::default()), + }) + } + + fn start(slf: &Bound<'_, Self>, message: String) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + if reporter.enabled { + return reporter.locked(py, || { + Self::close(slf)?; + reporter.state().message = Some(message); + Self::enter_status(slf)?; + claim( + reporter.lines.bind(py), + reporter.stream.bind(py), + slf.as_any(), + )?; + reporter.state().active = true; + Ok(()) + }); + } + reporter.state().active = true; + reporter.write_message(py, &message, true) + } + + fn update(&self, py: Python<'_>, message: String) -> PyResult<()> { + if self.enabled { + let updated: bool = self.locked(py, || { + let drawn: bool = { + let mut state = self.state(); + if state.message.is_none() { + return Ok(false); + } + state.message = Some(message.clone()); + state.drawn + }; + if drawn { + self.draw(py)?; + } + Ok(true) + })?; + if updated { + return Ok(()); + } + } + self.write_message(py, &message, true) + } + + #[pyo3(signature = (*, message, blank_line_after = false))] + fn complete(slf: &Bound<'_, Self>, message: &str, blank_line_after: bool) -> PyResult<()> { + let py: Python<'_> = slf.py(); + Self::close(slf)?; + slf.get().write_message(py, message, true)?; + if blank_line_after { + slf.get().write_blank_line(py)?; + } + Ok(()) + } + + /// Complete with a pre-styled message written verbatim. + #[pyo3(signature = (*, message, blank_line_after = false))] + fn complete_styled( + slf: &Bound<'_, Self>, + message: &str, + blank_line_after: bool, + ) -> PyResult<()> { + Self::close(slf)?; + let blank: &str = if blank_line_after { "\n" } else { "" }; + slf.get() + .write_persistent(slf.py(), &format!("{message}\n{blank}")) + } + + fn error(slf: &Bound<'_, Self>, message: &str) -> PyResult<()> { + Self::close(slf)?; + slf.get().write_message(slf.py(), message, false) + } + + fn close(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + reporter.locked(py, || { + release(reporter.lines.bind(py), slf.as_any())?; + reporter.exit_status(py)?; + let mut state = reporter.state(); + state.message = None; + state.active = false; + Ok(()) + }) + } + + /// Erase the live status so a persistent line can be written in its place. + fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.exit_status(py) + } + + /// Restart the live status below persistent output written while it was active. + fn redraw_transient_line(slf: &Bound<'_, Self>) -> PyResult<()> { + let redraw: bool = { + let state = slf.get().state(); + state.message.is_some() && !state.drawn + }; + if redraw { + Self::enter_status(slf)?; + } + Ok(()) + } + + /// Render one preflight progress update. + fn report_preflight_progress(slf: &Bound<'_, Self>, message: String) -> PyResult<()> { + if message.starts_with(PREFLIGHT_COMPLETION_PREFIX) { + return Self::complete(slf, &message, true); + } + if !slf.get().state().active { + return Self::start(slf, message); + } + slf.get().update(slf.py(), message) + } + + fn write_blank_line(&self, py: Python<'_>) -> PyResult<()> { + self.write_persistent(py, "\n") + } + + fn _spin_until_stopped(&self, py: Python<'_>, stop_event: &Bound<'_, PyAny>) -> PyResult<()> { + while !stop_event + .call_method1("wait", (SPINNER_TICK_SECONDS,))? + .is_truthy()? + { + self.locked(py, || { + if stop_event.call_method0("is_set")?.is_truthy()? { + return Ok(()); + } + self.draw(py) + })?; + } + Ok(()) + } +} + +/// A native status reporter writing to `stream`. +pub(crate) fn status_reporter<'py>( + stream: &Bound<'py, PyAny>, + use_color: bool, +) -> PyResult> { + let reporter: TransientStatusReporter = + TransientStatusReporter::new(stream.py(), stream, use_color, true)?; + Ok(Bound::new(stream.py(), reporter)?.into_any()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs index e83139b0f1..f73973bdc3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs @@ -10,13 +10,12 @@ use sqlbuild_core::progress::main::connection_phase_line::connection_phase_line; use sqlbuild_core::progress::main::connection_start_message::connection_start_message; use sqlbuild_core::progress::main::is_planning_completion_message::is_planning_completion_message; +use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::cli::progress::{ NativeProgressProjector, current_projector, projector_operation_active, write_persistent, }; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; -const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; -const STATUS_REPORTER_CLASS: &str = "TransientStatusReporter"; const CONNECTION_OPTIONS: (&[&str], &[&str]) = ( &["adapter_name", "stream"], &[ @@ -133,13 +132,7 @@ impl PlanningProgressReporter { #[new] #[pyo3(signature = (*, stream, use_color = false))] fn new(py: Python<'_>, stream: &Bound<'_, PyAny>, use_color: bool) -> PyResult { - let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("stream", stream)?; - keywords.set_item("use_color", use_color)?; - let status: Bound<'_, PyAny> = py - .import(STATUS_REPORTER_MODULE)? - .getattr(STATUS_REPORTER_CLASS)? - .call((), Some(&keywords))?; + let status: Bound<'_, PyAny> = status_reporter(stream, use_color)?; Ok(Self { status: status.unbind(), active: AtomicBool::new(false), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index f59594105c..d0d189dce9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -4,6 +4,7 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyBool, PyDict, PyDictMethods, PyMapping, PyString, PyStringMethods, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::compile_command::report::{ format_compile_json, format_compile_text, json_semantic_notice, }; @@ -22,7 +23,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const REUSE_MODELS_MODULE: &str = "sqlbuild.cli.compile_reuse.models"; const REUSE_TYPES_MODULE: &str = "sqlbuild.cli.compile_reuse.types"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; @@ -128,16 +128,7 @@ fn compile_status<'py>(request: &Bound<'py, PyAny>) -> PyResult { impl<'py> Progress<'py> { fn status(&self) -> PyResult> { - let py: Python<'py> = self.stream.py(); - call_with( - &imported(py, STATUS_REPORTER_MODULE, "TransientStatusReporter")?, - &[ - ("stream", self.stream.clone()), - ( - "use_color", - PyBool::new(py, self.use_color).to_owned().into_any(), - ), - ], - ) + status_reporter(&self.stream, self.use_color) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs index 01348c309c..ea7f60a7c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs @@ -3,10 +3,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyBool; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::compile_driver::python_calls::imported; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; /// `sys.stdout` or `sys.stderr`, looked up when written. pub(crate) fn stream<'py>(py: Python<'py>, name: &str) -> PyResult> { @@ -45,12 +45,5 @@ pub(crate) fn status<'py>( stream: &Bound<'py, PyAny>, use_color: bool, ) -> PyResult> { - let py: Python<'py> = stream.py(); - call_with( - &imported(py, STATUS_REPORTER_MODULE, "TransientStatusReporter")?, - &[ - ("stream", stream.clone()), - ("use_color", flag(py, use_color)), - ], - ) + status_reporter(stream, use_color) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs index c0707d7764..c32f91ad2d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs @@ -15,6 +15,7 @@ use sqlbuild_core::format_command::models::LintReport; use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_null_candidates; use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -26,7 +27,6 @@ use crate::bindings::_helpers::format_command::selection::{LintInputs, resolve_l use crate::bindings::_helpers::frontier::encoder::emit_error; const NATIVE_MODULE: &str = "sqlbuild._native"; -const STATUS_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; const DEFAULT_DIALECT: &str = "generic"; @@ -166,11 +166,8 @@ fn render_diff(py: Python<'_>, changes: &[FormatChange<'_>]) -> PyResult<()> { Ok(()) } -fn status_reporter<'py>(py: Python<'py>, use_color: bool) -> PyResult> { - let options: Bound<'py, PyDict> = PyDict::new(py); - options.set_item("stream", py.import("sys")?.getattr("stderr")?)?; - options.set_item("use_color", use_color)?; - imported(py, STATUS_MODULE, "TransientStatusReporter")?.call((), Some(&options)) +fn stderr_status<'py>(py: Python<'py>, use_color: bool) -> PyResult> { + status_reporter(&py.import("sys")?.getattr("stderr")?, use_color) } fn format_with_status<'py>( @@ -179,7 +176,7 @@ fn format_with_status<'py>( use_color: bool, ) -> PyResult<(FormatOutcome<'py>, Bound<'py, PyAny>, f64)> { let started: Instant = Instant::now(); - let status: Bound<'py, PyAny> = status_reporter(py, use_color)?; + let status: Bound<'py, PyAny> = stderr_status(py, use_color)?; status.call_method1("start", (START_MESSAGE,))?; match run_format(request) { Ok(outcome) => Ok((outcome, status, started.elapsed().as_secs_f64())), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs index 8cfc8ae1cc..09dc56013d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs @@ -33,6 +33,7 @@ use sqlbuild_refactor::refactoring::models::{ use sqlbuild_refactor::refactoring::types::Originals; use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_guard}; +use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::lineage::lineage_index::lineage_key; use crate::bindings::_helpers::refactor::refactor_facts::{ declaration_placement, model_facts, refactor_facts, @@ -41,7 +42,6 @@ use crate::bindings::_helpers::refactor::refactor_facts::{ const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const REFACTOR_ERRORS_MODULE: &str = "sqlbuild.compiler.refactoring.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const STATUS_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; const COMPILE_MODULE: &str = "sqlbuild.cli.commands._helpers.refactor.compile_facts"; const STAGING_PREFIX: &str = "sqb-refactor-"; const MOVE_COMMAND: &str = "mv"; @@ -430,12 +430,8 @@ fn run_refactor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult< let stream: Bound<'_, PyAny> = py.import("sys")? .getattr(if json_output { "stderr" } else { "stdout" })?; - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("stream", stream)?; - options.set_item("use_color", use_color)?; let status: Status<'_> = Status { - reporter: imported(py, STATUS_MODULE, "TransientStatusReporter")? - .call((), Some(&options))?, + reporter: status_reporter(&stream, use_color)?, }; let result: PyResult> = compiler_guard(|| run(py, &command, &status)); status.close()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs index 5c8be6460d..89fd26f640 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs @@ -12,6 +12,7 @@ use sqlbuild_core::scenario_command::main::render_snapshot_sync_result::render_s use sqlbuild_core::scenario_command::main::scenario_activity_message::scenario_activity_message; use sqlbuild_core::scenario_command::models::ScenarioNamespace; +use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::scenario_command::conversions::{ capture_row, scenario_result_row, write_local_runtime_target, write_warehouse_runtime_target, @@ -20,7 +21,6 @@ use crate::bindings::_helpers::scenario_command::setup::{ NATIVE_MODULE, boolean, flush, stream_isatty, text, write, }; -const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; /// How a finished scenario is reported. @@ -202,19 +202,12 @@ pub(crate) fn begin_section<'py>( (header, preamble, activity): (&str, &str, String), style: CliStyle, ) -> PyResult> { - let py: Python<'py> = stream.py(); write( stream, &format!("\n{}\n\n{preamble}", style.success_strong(header)), )?; flush(stream)?; - let status: Bound<'py, PyAny> = call_with( - &imported(py, STATUS_REPORTER_MODULE, "TransientStatusReporter")?, - &[ - ("stream", stream.clone()), - ("use_color", boolean(py, style.use_color)), - ], - )?; + let status: Bound<'py, PyAny> = status_reporter(stream, style.use_color)?; let status_is_tty: bool = stream_isatty(stream)?; if !status_is_tty { write(stream, &format!("{activity}\n\n"))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 5ecaaaa904..c28e0f40d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -23,6 +23,7 @@ use sqlbuild_core::test_command::models::{ TestFailure, TestInspectionEntry, TestParameter, TestPlanWarning, }; +use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -37,7 +38,6 @@ const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const TEST_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._sql_test"; const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; const PROGRESS_MODELS: &str = "sqlbuild.cli.progress.models"; -const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; const EXECUTOR_PIPELINE_MODELS: &str = "sqlbuild.executor.pipeline.models"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const RESOURCE_ID_MODULE: &str = "sqlbuild.executor.testing.main._resource_id"; @@ -520,10 +520,7 @@ fn run_tests<'py>( ("use_color", use_color.clone()), ], )?; - let preflight: Bound<'py, PyAny> = call_with( - &imported(py, STATUS_REPORTER_MODULE, "TransientStatusReporter")?, - &[("stream", stream), ("use_color", use_color)], - )?; + let preflight: Bound<'py, PyAny> = status_reporter(&stream, use_color.is_truthy()?)?; let event_writer: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "ExecutionEventWriter")?.call0()?; let relay: Bound<'py, TestCallbackRelay> = Bound::new( diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5c094dc9cd..d2ae957258 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -290,7 +290,6 @@ from sqlbuild.integrations.dbt.types import DbtInteropCommand from sqlbuild.lint.models import HeaderSpan, LintBody, LintConfig, LintRunResult, LintViolation from sqlbuild.lint.types import RelationKeyRow from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter from sqlbuild.presentation.models import DisplayOptions from sqlbuild.presentation.types import CompletionState from sqlbuild.python_nodes.models import SqlResourceRef @@ -3265,6 +3264,19 @@ def resolve_loader_defaults( cli_vars: dict[str, object] | None = None, ) -> tuple[str | None, str | None]: ... +class TransientStatusReporter: + def __init__(self, *, stream: TextIO, use_color: bool = False, enabled: bool = True) -> None: ... + def start(self, message: str) -> None: ... + def update(self, message: str) -> None: ... + def complete(self, *, message: str, blank_line_after: bool = False) -> None: ... + def complete_styled(self, *, message: str, blank_line_after: bool = False) -> None: ... + def error(self, message: str) -> None: ... + def close(self) -> None: ... + def clear_transient_line(self) -> None: ... + def redraw_transient_line(self) -> None: ... + def report_preflight_progress(self, message: str) -> None: ... + def write_blank_line(self) -> None: ... + class LoadProgressReporter: def __init__( self, *, stream: TextIO, use_color: bool, source_order: dict[str, int], total_count: int diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py index 531dfe13fe..2a0a9f7fc9 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py @@ -5,6 +5,7 @@ import sys import time +from sqlbuild._native import TransientStatusReporter from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import ( JanitorCompileContext, @@ -15,7 +16,6 @@ ) from sqlbuild.executor.janitor.main.plan import build_janitor_plan from sqlbuild.executor.janitor.models import JanitorDirectModeSettings, JanitorPlan -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/cli/commands/classes/dbt_init_progress_reporter.py b/src/sqlbuild/cli/commands/classes/dbt_init_progress_reporter.py index e8b39b8c31..ce30fe04a4 100644 --- a/src/sqlbuild/cli/commands/classes/dbt_init_progress_reporter.py +++ b/src/sqlbuild/cli/commands/classes/dbt_init_progress_reporter.py @@ -5,7 +5,7 @@ import time from typing import TextIO -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter +from sqlbuild._native import TransientStatusReporter class DbtInitProgressReporter: diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py index 70ac8c252b..bd8cf49d2d 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py @@ -9,6 +9,7 @@ from typing import TextIO, cast from sqlbuild._native import ( + TransientStatusReporter, dbt_execution_ls_argv, parse_dbt_ls_json_lines, render_dbt_node_result, @@ -27,7 +28,6 @@ DbtNodeMessage, ) from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter _COMMAND_AND_SUBCOMMAND_ARGUMENT_COUNT: int = 2 diff --git a/src/sqlbuild/integrations/dbt/_helpers/runtime/event_stream.py b/src/sqlbuild/integrations/dbt/_helpers/runtime/event_stream.py index adfd965658..754fc6c7ca 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/runtime/event_stream.py +++ b/src/sqlbuild/integrations/dbt/_helpers/runtime/event_stream.py @@ -14,11 +14,11 @@ from sqlbuild._native import ( DbtEventStream, DbtEventStreamUpdate, + TransientStatusReporter, format_dbt_active_node_status, ) from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError from sqlbuild.integrations.dbt.models import DbtNodeExecutionResult -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle _DBT_STATUS_REFRESH_SECONDS: float = 1.0 diff --git a/src/sqlbuild/presentation/classes/transient_status_reporter.py b/src/sqlbuild/presentation/classes/transient_status_reporter.py deleted file mode 100644 index 9a3aa9e5e9..0000000000 --- a/src/sqlbuild/presentation/classes/transient_status_reporter.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Transient status reporter class.""" - -from __future__ import annotations - -import atexit -from contextlib import AbstractContextManager -from typing import TYPE_CHECKING, TextIO - -from sqlbuild.errors.contracts.exceptions import SharedInputError -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator -from sqlbuild.presentation.main._progress_spinners_disabled import progress_spinners_disabled -from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator - -if TYPE_CHECKING: - from rich.console import Console - from rich.status import Status - - -class TransientStatusReporter: - """Render one transient status line with optional spinner updates.""" - - def __init__( - self, - *, - stream: TextIO, - use_color: bool = False, - enabled: bool = True, - ) -> None: - self._stream: TextIO = stream - self._lines: TransientLineCoordinator = shared_transient_line_coordinator() - self._style: CliStyle = CliStyle(use_color=use_color) - self._enabled: bool = ( - enabled - and not progress_spinners_disabled() - and hasattr(stream, "isatty") - and stream.isatty() - ) - self._console: Console | None = None - if self._enabled: - from rich.console import Console - - self._console = Console(file=stream, no_color=(not use_color)) - self._status_context: AbstractContextManager[Status] | None = None - self._status: Status | None = None - self._active: bool = False - self._message: str | None = None - atexit.register(self.close) - - def start(self, message: str) -> None: - if self._enabled: - with self._lines.lock: - self.close() - self._message = message - self._enter_status() - self._lines.claim(stream=self._stream, owner=self) - self._active = True - return - self._active = True - self._write_message(message=message, dim_output=True) - - def update(self, message: str) -> None: - if self._enabled: - with self._lines.lock: - if self._message is not None: - self._message = message - if self._status is not None: - self._status.update(message) - return - self._write_message(message=message, dim_output=True) - - def complete(self, *, message: str, blank_line_after: bool = False) -> None: - self.close() - self._write_message(message=message, dim_output=True) - if blank_line_after: - self.write_blank_line() - - def complete_styled(self, *, message: str, blank_line_after: bool = False) -> None: - """Complete with a pre-styled message written verbatim.""" - - self.close() - self._lines.write_persistent( - stream=self._stream, text=f"{message}\n" + ("\n" if blank_line_after else "") - ) - - def error(self, message: str) -> None: - self.close() - self._write_message(message=message, dim_output=False) - - def close(self) -> None: - with self._lines.lock: - self._lines.release(owner=self) - self._exit_status() - self._message = None - self._active = False - - def clear_transient_line(self) -> None: - """Erase the live status so a persistent line can be written in its place.""" - - self._exit_status() - - def redraw_transient_line(self) -> None: - """Restart the live status below persistent output written while it was active.""" - - if self._message is not None and self._status_context is None: - self._enter_status() - - def report_preflight_progress(self, message: str) -> None: - """Render one preflight progress update.""" - - if message.startswith("Prepared "): - self.complete(message=message, blank_line_after=True) - return - if not self._active: - self.start(message) - return - self.update(message) - - def write_blank_line(self) -> None: - self._lines.write_persistent(stream=self._stream, text="\n") - - def _enter_status(self) -> None: - if self._console is None or self._message is None: - raise SharedInputError( - "status reporter was enabled without an initialized console", - code="G001", - ) - self._status_context = self._console.status(status=self._message, spinner="dots") - self._status = self._status_context.__enter__() - - def _exit_status(self) -> None: - if self._status_context is not None: - self._status_context.__exit__(None, None, None) - self._status_context = None - self._status = None - - def _write_message(self, *, message: str, dim_output: bool) -> None: - formatted_message: str = self._style.muted(message) if dim_output else message - self._lines.write_persistent(stream=self._stream, text=f"{formatted_message}\n") diff --git a/src/sqlbuild/presentation/main/_maybe_status.py b/src/sqlbuild/presentation/main/_maybe_status.py index 3e12846418..d9cb7214cd 100644 --- a/src/sqlbuild/presentation/main/_maybe_status.py +++ b/src/sqlbuild/presentation/main/_maybe_status.py @@ -6,8 +6,8 @@ from collections.abc import Iterator from contextlib import contextmanager +from sqlbuild._native import TransientStatusReporter from sqlbuild.presentation._helpers.progress import progress_spinners_disabled -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter @contextmanager diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index c48c7dfd15..0c85d6fe57 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -1,5 +1,6 @@ from __future__ import annotations +import io from collections.abc import Callable from pathlib import Path from typing import Any @@ -63,6 +64,13 @@ ) +class TtyStdout(io.StringIO): + """A captured stdout that reports a terminal.""" + + def isatty(self) -> bool: + return True + + class NoConnectDuckDbAdapter(DuckDbAdapter): """DuckDB adapter test double that fails if compile opens a connection.""" diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 4572ca2eca..4b6351bb72 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -19,7 +19,6 @@ from sqlbuild.compiler.planner.main.execution import ( _plan_compiled_sql_test_artifacts as plan_artifacts_module, ) -from sqlbuild.presentation.classes import transient_status_reporter from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileColumnContractModeTestCase, CompileCommandTestCase, @@ -32,6 +31,7 @@ ) from tests.unit.src.sqlbuild.cli.commands.main.compile.helpers import ( NoConnectDuckDbAdapter, + TtyStdout, prepare_python_compile_project, prepare_rule_gated_compile_project, prepare_static_compile_project, @@ -101,38 +101,9 @@ def test_given_tty_stdout_when_running_compile_then_it_persists_phase_timings( monkeypatch: pytest.MonkeyPatch, ) -> None: project_dir: Path = prepare_static_compile_project(tmp_path) - started_messages: list[str] = [] - completed_messages: list[str] = [] - - class FakeStdout: - def isatty(self) -> bool: - return True - - def write(self, text: str) -> int: - return len(text) - - def flush(self) -> None: - return None - - class FakeStatusReporter: - def __init__(self, **kwargs: object) -> None: - del kwargs - - def start(self, message: str) -> None: - started_messages.append(message) - - def update(self, message: str) -> None: - started_messages.append(message) - - def complete(self, message: str, *, blank_line_after: bool = False) -> None: - del blank_line_after - completed_messages.append(message) - - def close(self) -> None: - return None - - monkeypatch.setattr(sys, "stdout", FakeStdout()) - monkeypatch.setattr(transient_status_reporter, "TransientStatusReporter", FakeStatusReporter) + stdout: TtyStdout = TtyStdout() + monkeypatch.setenv("SQLBUILD_NO_PROGRESS", "1") + monkeypatch.setattr(sys, "stdout", stdout) monkeypatch.setattr( runtime_adapters, "resolve_adapter", @@ -140,21 +111,22 @@ def close(self) -> None: ) exit_code: int = run_compile( - CompileCommandRequest(project_dir=project_dir, no_sql_validation=True) + CompileCommandRequest(project_dir=project_dir, no_sql_validation=True, no_color=True) ) + rendered_stdout: str = stdout.getvalue() assert exit_code == test_case.expected_exit_code - assert started_messages == [ + for message in ( "Discovering project...", "Compiling project graph...", "Analyzing column lineage...", "Validating model contracts...", "Evaluating built-in and custom rules...", "Writing compiled artifacts...", - ] - assert len(completed_messages) == len(test_case.expected_stdout_fragments) - for index, fragment in enumerate(test_case.expected_stdout_fragments): - assert completed_messages[index].startswith(f"{fragment} (") + ): + assert f"{message}\n" in rendered_stdout + for fragment in test_case.expected_stdout_fragments: + assert f"\n{fragment} (" in rendered_stdout @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_event_stream.py b/tests/unit/src/sqlbuild/integrations/dbt/test_event_stream.py index dc27cf2af1..eec7cdfffe 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_event_stream.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_event_stream.py @@ -888,21 +888,6 @@ def test_given_active_dbt_node_when_dbt_stream_is_silent_then_status_elapsed_upd test_case: DbtSilentStatusRefreshTestCase, monkeypatch: pytest.MonkeyPatch, ) -> None: - status_messages: list[str] = [] - - class CapturingStatusReporter: - def __init__(self, *args: object, **kwargs: object) -> None: - pass - - def start(self, message: str) -> None: - status_messages.append(message) - - def update(self, message: str) -> None: - status_messages.append(message) - - def close(self) -> None: - pass - class TtyStream(io.StringIO): def isatty(self) -> bool: return True @@ -934,10 +919,7 @@ class StubProcess: def wait(self) -> int: return 0 - monkeypatch.setattr( - "sqlbuild.integrations.dbt._helpers.runtime.event_stream.TransientStatusReporter", - CapturingStatusReporter, - ) + monkeypatch.setenv("SQLBUILD_NO_PROGRESS", "1") monkeypatch.setattr( "sqlbuild.integrations.dbt._helpers.runtime.event_stream._DBT_STATUS_REFRESH_SECONDS", test_case.refresh_seconds, @@ -947,18 +929,20 @@ def wait(self) -> int: lambda *args, **kwargs: StubProcess(), ) + stream: TtyStream = TtyStream() returncode: int returncode, _ = execute_dbt_json_event_stream( argv=("dbt", "run"), cwd=None, - stream=TtyStream(), + stream=stream, use_color=False, target_path=None, ) + status_lines: list[str] = stream.getvalue().splitlines() assert returncode == 0 - assert test_case.expected_initial_status in status_messages - assert test_case.expected_refreshed_status in status_messages + assert test_case.expected_initial_status in status_lines + assert test_case.expected_refreshed_status in status_lines @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/presentation/classes/test_status.py b/tests/unit/src/sqlbuild/presentation/classes/test_status.py index 2defadc1c5..4e5b268164 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/test_status.py +++ b/tests/unit/src/sqlbuild/presentation/classes/test_status.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter +from sqlbuild._native import TransientStatusReporter from sqlbuild.presentation.main._progress_spinners_disabled import progress_spinners_disabled from tests.unit.src.sqlbuild.presentation.classes._test_types import ( ProgressSpinnersDisabledTestCase, @@ -116,6 +116,7 @@ def test_given_no_progress_env_when_building_status_on_tty_then_toggles_spinner( stream: _TtyStream = _TtyStream() reporter: TransientStatusReporter = TransientStatusReporter(stream=stream, use_color=False) + reporter.start("Working...") + reporter.close() - spinner_enabled: bool = reporter._enabled - assert spinner_enabled == (not test_case.expected_disabled) + assert ("Working...\n" in stream.getvalue()) == test_case.expected_disabled From b1765fedd33100e8a8472a74a183e2858bf52584 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 04:44:01 +0100 Subject: [PATCH 19/40] refactor: move the status spinner line into progress and cover the native reporter --- crates/sqlbuild-core/src/cli_text/main/mod.rs | 1 - .../src/cli_text/tests/test_cli_text.rs | 44 +------------ .../src/cli_text/tests/test_types.rs | 8 --- .../sqlbuild-core/src/progress/constants.rs | 2 + .../main/format_status_spinner_line.rs | 6 +- crates/sqlbuild-core/src/progress/main/mod.rs | 1 + .../src/progress/tests/test_progress_text.rs | 44 ++++++++++++- .../src/progress/tests/test_types.rs | 9 +++ .../build_progress/transient_status.rs | 2 +- src/sqlbuild/_native.pyi | 4 +- .../presentation/classes/_test_types.py | 8 +++ .../sqlbuild/presentation/classes/helpers.py | 24 ++++++++ .../presentation/classes/test_status.py | 61 ++++++++++++++++++- 13 files changed, 155 insertions(+), 59 deletions(-) rename crates/sqlbuild-core/src/{cli_text => progress}/main/format_status_spinner_line.rs (83%) diff --git a/crates/sqlbuild-core/src/cli_text/main/mod.rs b/crates/sqlbuild-core/src/cli_text/main/mod.rs index b6aced55d8..60ec06a502 100644 --- a/crates/sqlbuild-core/src/cli_text/main/mod.rs +++ b/crates/sqlbuild-core/src/cli_text/main/mod.rs @@ -12,7 +12,6 @@ pub mod format_phase_line; pub mod format_plan_completion; pub mod format_plan_text; pub mod format_status_cell; -pub mod format_status_spinner_line; pub mod format_summary_footer; pub mod format_surface_header; pub mod format_tree_connector; diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index ff0596afc9..ddac7563c2 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -3,7 +3,6 @@ use crate::cli_text::main::format_compile_text::format_compile_text; use crate::cli_text::main::format_dbt_interop_plan::format_dbt_interop_plan; use crate::cli_text::main::format_plan_completion::format_plan_completion; use crate::cli_text::main::format_plan_text::format_plan_text; -use crate::cli_text::main::format_status_spinner_line::format_status_spinner_line; use crate::cli_text::main::semantic_coverage_notice::semantic_coverage_notice; use crate::cli_text::models::{ CliStyle, CompileTextReport, DbtInteropPlanText, DbtLsNodeText, DiagnosticText, DisplayLimit, @@ -12,7 +11,7 @@ use crate::cli_text::models::{ use crate::cli_text::tests::helpers::{diagnostic, location, plan_report, table_model}; use crate::cli_text::tests::test_types::{ CompileTextTestCase, DbtPlanTextTestCase, DiagnosticsTextTestCase, NoticeTestCase, - PlanTextTestCase, StatusSpinnerLineTestCase, + PlanTextTestCase, }; /// Expectations recorded from the retired Python formatter. @@ -233,44 +232,3 @@ fn given_dbt_interop_plan_when_formatting_then_matches_python_text() { ); } } - -#[test] -fn given_status_message_when_rendering_spinner_line_then_fits_terminal() { - let test_cases = [ - StatusSpinnerLineTestCase { - description: "short message is kept with a green frame", - message: "Compiling project graph...", - columns: 80, - style: CliStyle { use_color: true }, - expected_line: "\x1b[32m\u{280b}\x1b[0m Compiling project graph...", - }, - StatusSpinnerLineTestCase { - description: "long message is cut below the terminal width", - message: "Compiling project graph...", - columns: 15, - style: CliStyle { use_color: false }, - expected_line: "\u{280b} Compiling...", - }, - StatusSpinnerLineTestCase { - description: "unknown width keeps only the first line", - message: "first line\nsecond line", - columns: 0, - style: CliStyle { use_color: false }, - expected_line: "\u{280b} first line", - }, - ]; - - for test_case in test_cases { - assert_eq!( - format_status_spinner_line( - "\u{280b}", - test_case.message, - test_case.columns, - test_case.style - ), - test_case.expected_line, - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_types.rs b/crates/sqlbuild-core/src/cli_text/tests/test_types.rs index e733365cb3..ed807492b8 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_types.rs @@ -32,11 +32,3 @@ pub(super) struct DbtPlanTextTestCase { pub(super) max_entries: Option, pub(super) expected_text: &'static str, } - -pub(super) struct StatusSpinnerLineTestCase { - pub(super) description: &'static str, - pub(super) message: &'static str, - pub(super) columns: usize, - pub(super) style: CliStyle, - pub(super) expected_line: &'static str, -} diff --git a/crates/sqlbuild-core/src/progress/constants.rs b/crates/sqlbuild-core/src/progress/constants.rs index 4338385c18..aa0ba9ec5e 100644 --- a/crates/sqlbuild-core/src/progress/constants.rs +++ b/crates/sqlbuild-core/src/progress/constants.rs @@ -108,3 +108,5 @@ pub(crate) const DBT_COMMAND: &str = "dbt"; pub(crate) const DBT_JSON_OPTION: &str = "--json"; pub(crate) const JSON_FORMAT: &str = "json"; pub(crate) const CSV_FORMAT: &str = "csv"; +/// What replaces the cut tail of a transient status message. +pub(crate) const STATUS_ELLIPSIS: &str = "..."; diff --git a/crates/sqlbuild-core/src/cli_text/main/format_status_spinner_line.rs b/crates/sqlbuild-core/src/progress/main/format_status_spinner_line.rs similarity index 83% rename from crates/sqlbuild-core/src/cli_text/main/format_status_spinner_line.rs rename to crates/sqlbuild-core/src/progress/main/format_status_spinner_line.rs index 8dea6ccb41..31dc33f57e 100644 --- a/crates/sqlbuild-core/src/cli_text/main/format_status_spinner_line.rs +++ b/crates/sqlbuild-core/src/progress/main/format_status_spinner_line.rs @@ -1,7 +1,7 @@ //! The live line of a transient status: a spinner frame and the current message. -use crate::cli_text::constants::ELLIPSIS; use crate::cli_text::models::CliStyle; +use crate::progress::constants::STATUS_ELLIPSIS; /// The frame and the first message line, cut to fit `columns` (`0` leaves it uncut). #[must_use] @@ -17,9 +17,9 @@ pub fn format_status_spinner_line( text.to_owned() } else { format!( - "{}{ELLIPSIS}", + "{}{STATUS_ELLIPSIS}", text.chars() - .take(width.saturating_sub(ELLIPSIS.len())) + .take(width.saturating_sub(STATUS_ELLIPSIS.len())) .collect::() ) }; diff --git a/crates/sqlbuild-core/src/progress/main/mod.rs b/crates/sqlbuild-core/src/progress/main/mod.rs index 67bb535223..9b70ca7b90 100644 --- a/crates/sqlbuild-core/src/progress/main/mod.rs +++ b/crates/sqlbuild-core/src/progress/main/mod.rs @@ -8,6 +8,7 @@ pub mod format_execution_header; pub mod format_expectation_detail; pub mod format_expectation_name; pub mod format_inspection_record; +pub mod format_status_spinner_line; pub mod is_planning_completion_message; pub mod machine_output_mode; pub mod progress_terminal_duration; diff --git a/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs b/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs index c6c45ce34d..4ef6de339e 100644 --- a/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs +++ b/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs @@ -5,6 +5,7 @@ use crate::progress::main::execution_header_text::execution_header_text; use crate::progress::main::format_expectation_detail::format_expectation_detail; use crate::progress::main::format_expectation_name::format_expectation_name; use crate::progress::main::format_inspection_record::format_inspection_record; +use crate::progress::main::format_status_spinner_line::format_status_spinner_line; use crate::progress::main::is_planning_completion_message::is_planning_completion_message; use crate::progress::main::machine_output_mode::machine_output_mode; use crate::progress::models::{ExpectationStep, InspectionRecord, OutputModeFacts}; @@ -12,7 +13,7 @@ use crate::progress::models::{ExpectationStep, InspectionRecord, OutputModeFacts const PLAIN: CliStyle = CliStyle { use_color: false }; use crate::progress::tests::test_types::{ CompletionMessageTestCase, ExpectationDetailTestCase, InspectionRecordTestCase, - OutputModeTestCase, ProgressLineTestCase, + OutputModeTestCase, ProgressLineTestCase, StatusSpinnerLineTestCase, }; #[test] @@ -253,3 +254,44 @@ fn given_parsed_options_when_choosing_output_mode_then_machine_output_is_detecte ); } } + +#[test] +fn given_status_message_when_rendering_spinner_line_then_fits_terminal() { + let test_cases = [ + StatusSpinnerLineTestCase { + description: "short message is kept with a green frame", + message: "Compiling project graph...", + columns: 80, + style: CliStyle { use_color: true }, + expected_line: "\x1b[32m\u{280b}\x1b[0m Compiling project graph...", + }, + StatusSpinnerLineTestCase { + description: "long message is cut below the terminal width", + message: "Compiling project graph...", + columns: 15, + style: CliStyle { use_color: false }, + expected_line: "\u{280b} Compiling...", + }, + StatusSpinnerLineTestCase { + description: "unknown width keeps only the first line", + message: "first line\nsecond line", + columns: 0, + style: CliStyle { use_color: false }, + expected_line: "\u{280b} first line", + }, + ]; + + for test_case in test_cases { + assert_eq!( + format_status_spinner_line( + "\u{280b}", + test_case.message, + test_case.columns, + test_case.style + ), + test_case.expected_line, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/progress/tests/test_types.rs b/crates/sqlbuild-core/src/progress/tests/test_types.rs index 48942a07ce..0a9e9d687d 100644 --- a/crates/sqlbuild-core/src/progress/tests/test_types.rs +++ b/crates/sqlbuild-core/src/progress/tests/test_types.rs @@ -1,3 +1,4 @@ +use crate::cli_text::models::CliStyle; use crate::progress::models::{ProgressEvent, ProgressSettings}; use crate::terminal_events::models::TerminalRequest; @@ -40,3 +41,11 @@ pub(super) struct OutputModeTestCase { pub(super) facts: crate::progress::models::OutputModeFacts, pub(super) expected_machine_output: bool, } + +pub(super) struct StatusSpinnerLineTestCase { + pub(super) description: &'static str, + pub(super) message: &'static str, + pub(super) columns: usize, + pub(super) style: CliStyle, + pub(super) expected_line: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs index de45f63a02..7b7cbcfa44 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs @@ -5,8 +5,8 @@ use std::sync::{Mutex, MutexGuard, PoisonError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyclass, pymethods}; -use sqlbuild_core::cli_text::main::format_status_spinner_line::format_status_spinner_line; use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::progress::main::format_status_spinner_line::format_status_spinner_line; use crate::bindings::_helpers::build_progress::terminal::{ CLEAR_LINE, HIDE_CURSOR, SHOW_CURSOR, SPINNER_FRAMES, SPINNER_TICK_SECONDS, claim, flush, diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index d2ae957258..668642efac 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3265,7 +3265,9 @@ def resolve_loader_defaults( ) -> tuple[str | None, str | None]: ... class TransientStatusReporter: - def __init__(self, *, stream: TextIO, use_color: bool = False, enabled: bool = True) -> None: ... + def __init__( + self, *, stream: TextIO, use_color: bool = False, enabled: bool = True + ) -> None: ... def start(self, message: str) -> None: ... def update(self, message: str) -> None: ... def complete(self, *, message: str, blank_line_after: bool = False) -> None: ... diff --git a/tests/unit/src/sqlbuild/presentation/classes/_test_types.py b/tests/unit/src/sqlbuild/presentation/classes/_test_types.py index bedaf3ccc2..b1b41c5a5a 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/presentation/classes/_test_types.py @@ -45,3 +45,11 @@ class TransientLineCoordinatorTestCase: description: str persistent_stream_is_tty: bool expected_events: tuple[str, ...] + + +@dataclass(frozen=True) +class StatusReporterOutputTestCase: + description: str + use_color: bool + tty: bool + expected_persistent_lines: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/presentation/classes/helpers.py b/tests/unit/src/sqlbuild/presentation/classes/helpers.py index 8551ac1c9e..b928748d8e 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/helpers.py +++ b/tests/unit/src/sqlbuild/presentation/classes/helpers.py @@ -1,12 +1,15 @@ from __future__ import annotations import io +import re from collections.abc import Callable from types import MappingProxyType from typing import cast import pytest +from sqlbuild._native import TransientStatusReporter + def _delete_no_progress_env(monkeypatch: pytest.MonkeyPatch, env_value: str | None) -> None: del env_value @@ -53,3 +56,24 @@ def clear_transient_line(self) -> None: def redraw_transient_line(self) -> None: self._events.append("redraw") + + +def drive_status_reporter(reporter: TransientStatusReporter) -> None: + """Run one start, update, completion, preflight and error sequence.""" + + reporter.start("Discovering project...") + reporter.update("Compiling project graph...") + reporter.complete(message="Compiled project graph. (0.1s)", blank_line_after=True) + reporter.report_preflight_progress("Resolving tests...") + reporter.report_preflight_progress("Prepared 2 tests") + reporter.start("Formatting SQL...") + reporter.error("Formatting SQL ERROR") + reporter.complete_styled(message="styled") + reporter.close() + + +def persistent_lines(text: str) -> tuple[str, ...]: + """The lines left after transient status lines are erased.""" + + without_cursor: str = re.sub(r"\x1b\[\?25[hl]", "", text) + return tuple(line.rsplit("\x1b[K", 1)[-1] for line in without_cursor.split("\n")) diff --git a/tests/unit/src/sqlbuild/presentation/classes/test_status.py b/tests/unit/src/sqlbuild/presentation/classes/test_status.py index 4e5b268164..c36113fa77 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/test_status.py +++ b/tests/unit/src/sqlbuild/presentation/classes/test_status.py @@ -8,8 +8,14 @@ from sqlbuild.presentation.main._progress_spinners_disabled import progress_spinners_disabled from tests.unit.src.sqlbuild.presentation.classes._test_types import ( ProgressSpinnersDisabledTestCase, + StatusReporterOutputTestCase, +) +from tests.unit.src.sqlbuild.presentation.classes.helpers import ( + RecordingStream, + apply_no_progress_env, + drive_status_reporter, + persistent_lines, ) -from tests.unit.src.sqlbuild.presentation.classes.helpers import apply_no_progress_env class _TtyStream(io.StringIO): @@ -120,3 +126,56 @@ def test_given_no_progress_env_when_building_status_on_tty_then_toggles_spinner( reporter.close() assert ("Working...\n" in stream.getvalue()) == test_case.expected_disabled + + +@pytest.mark.parametrize( + "test_case", + ( + StatusReporterOutputTestCase( + description="plain stream writes every message as a muted line", + use_color=True, + tty=False, + expected_persistent_lines=( + "\x1b[2mDiscovering project...\x1b[0m", + "\x1b[2mCompiling project graph...\x1b[0m", + "\x1b[2mCompiled project graph. (0.1s)\x1b[0m", + "", + "\x1b[2mResolving tests...\x1b[0m", + "\x1b[2mPrepared 2 tests\x1b[0m", + "", + "\x1b[2mFormatting SQL...\x1b[0m", + "Formatting SQL ERROR", + "styled", + "", + ), + ), + StatusReporterOutputTestCase( + description="terminal erases live status and keeps only completions", + use_color=False, + tty=True, + expected_persistent_lines=( + "Compiled project graph. (0.1s)", + "", + "Prepared 2 tests", + "", + "Formatting SQL ERROR", + "styled", + "", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_status_sequence_when_reporting_then_writes_expected_persistent_lines( + test_case: StatusReporterOutputTestCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("SQLBUILD_NO_PROGRESS", raising=False) + stream: RecordingStream = RecordingStream(events=[], tty=test_case.tty) + reporter: TransientStatusReporter = TransientStatusReporter( + stream=stream, use_color=test_case.use_color + ) + + drive_status_reporter(reporter) + + assert persistent_lines(stream.getvalue()) == test_case.expected_persistent_lines From bd15466efed96e3b8c1441bba309d3abf9f66dfc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 04:47:58 +0100 Subject: [PATCH 20/40] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index b91c06bd6b..732de14f93 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 58013 +remaining_lines = 57874 [[stays]] path = "executor" From bfb2e2fda8cca3794727b1cb0feb5c158de8b918 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 04:51:58 +0100 Subject: [PATCH 21/40] refactor: drop the empty compile config helper modules --- python-boundary.toml | 2 +- .../compiler/compile/_helpers/config/dynamic_columns.py | 3 --- .../compiler/compile/_helpers/config/model_validation.py | 3 --- 3 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 src/sqlbuild/compiler/compile/_helpers/config/dynamic_columns.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/config/model_validation.py diff --git a/python-boundary.toml b/python-boundary.toml index 732de14f93..1183af3b8b 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 57874 +remaining_lines = 57868 [[stays]] path = "executor" diff --git a/src/sqlbuild/compiler/compile/_helpers/config/dynamic_columns.py b/src/sqlbuild/compiler/compile/_helpers/config/dynamic_columns.py deleted file mode 100644 index 9978f375c3..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/config/dynamic_columns.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Dynamic column-family configuration parsing for model headers.""" - -from __future__ import annotations diff --git a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py deleted file mode 100644 index 435a4ef422..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compile-time validation of model config combinations.""" - -from __future__ import annotations From 67f867f4bf79e089c29118efe07bb698068d394d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 05:13:35 +0100 Subject: [PATCH 22/40] refactor: detect planner model changes natively --- crates/sqlbuild-analysis/src/lib.rs | 1 + .../_helpers/function_hashes.rs | 47 + .../planner_changes/_helpers/json_payload.rs | 80 ++ .../src/planner_changes/_helpers/mod.rs | 6 + .../_helpers/pick_more_aggressive.rs | 20 + .../_helpers/rename_mappings.rs | 63 ++ .../_helpers/schema_comparer.rs | 159 ++++ .../with_origin_cursor_input_names.rs | 37 + .../src/planner_changes/constants.rs | 26 + .../src/planner_changes/errors.rs | 12 + .../main/detect_schema_changes.rs | 45 + .../planner_changes/main/metadata_changes.rs | 37 + .../src/planner_changes/main/mod.rs | 11 + .../planner_changes/main/model_backfill.rs | 27 + .../planner_changes/main/model_change_kind.rs | 18 + .../main/origin_reference_names.rs | 54 ++ .../main/recorded_declared_columns_hash.rs | 17 + .../main/replay_policy_needed.rs | 9 + .../main/resolve_replay_on_change.rs | 24 + .../main/version_identity_payload.rs | 35 + .../main/with_declared_columns_hash.rs | 20 + .../main/without_declared_columns_hash.rs | 15 + .../src/planner_changes/mod.rs | 11 + .../src/planner_changes/models.rs | 108 +++ .../src/planner_changes/tests.rs | 6 + .../src/planner_changes/tests/helpers.rs | 46 + .../tests/test_planner_changes.rs | 427 +++++++++ .../src/planner_changes/tests/test_types.rs | 50 ++ .../src/planner_changes/types.rs | 23 + .../src/bindings/_helpers/mod.rs | 1 + .../_helpers/planner_changes/actions.rs | 61 ++ .../_helpers/planner_changes/detect.rs | 820 ++++++++++++++++++ .../_helpers/planner_changes/metadata.rs | 46 + .../bindings/_helpers/planner_changes/mod.rs | 6 + .../_helpers/planner_changes/registration.rs | 12 + .../_helpers/planner_changes/renames.rs | 275 ++++++ .../_helpers/planner_changes/results.rs | 109 +++ .../src/bindings/main/register.rs | 7 +- src/sqlbuild/_native.pyi | 78 +- .../planner/_helpers/changes/__init__.py | 1 - .../planner/_helpers/changes/actions.py | 32 - .../planner/_helpers/changes/detect.py | 375 -------- .../planner/_helpers/changes/metadata.py | 146 ---- .../planner/_helpers/changes/policy.py | 48 - .../planner/_helpers/changes/query.py | 15 - .../_helpers/changes/recorded_renames.py | 75 -- .../_helpers/changes/reference_renames.py | 107 --- .../planner/_helpers/changes/schema.py | 237 ----- .../_helpers/migrations/compatibility.py | 3 +- .../_helpers/migrations/fingerprint.py | 2 +- .../planner/_helpers/migrations/planning.py | 3 +- .../planner/_helpers/output/plan_entry.py | 6 +- .../planner/_helpers/planning/identities.py | 2 +- .../_helpers/pruning/selection_staleness.py | 2 +- .../classes/migration_fingerprint_cache.py | 2 +- .../planner/main/changes/_model_changes.py | 34 - .../main/changes/relation_replacement.py | 16 - .../planner/main/execution/execution.py | 3 +- .../executor/build/_helpers/scheduler.py | 9 +- .../_helpers/materializations/table_cursor.py | 4 +- .../compiler/planner/main/_test_types.py | 1 - .../planner/main/test_build_execution_plan.py | 21 +- .../_helpers/test_module_paths.py | 1 + .../planner/_helpers/changes/_test_types.py | 22 - .../changes/test_declared_columns_metadata.py | 7 +- .../planner/_helpers/changes/test_detect.py | 2 +- .../planner/_helpers/changes/test_policy.py | 61 -- .../planner/_helpers/changes/test_query.py | 54 -- .../changes/test_reference_renames.py | 3 +- .../planner/_helpers/changes/test_schema.py | 3 +- 70 files changed, 2872 insertions(+), 1274 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_changes/_helpers/function_hashes.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/_helpers/json_payload.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/_helpers/pick_more_aggressive.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/_helpers/rename_mappings.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/_helpers/schema_comparer.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/_helpers/with_origin_cursor_input_names.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/constants.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/errors.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/main/detect_schema_changes.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/main/metadata_changes.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/main/model_backfill.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/main/model_change_kind.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/main/origin_reference_names.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/main/recorded_declared_columns_hash.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/main/replay_policy_needed.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/main/resolve_replay_on_change.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/main/with_declared_columns_hash.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/main/without_declared_columns_hash.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/models.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/tests.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/tests/test_planner_changes.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/tests/test_types.rs create mode 100644 crates/sqlbuild-analysis/src/planner_changes/types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_changes/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_changes/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/changes/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/changes/actions.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/changes/detect.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/changes/metadata.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/changes/policy.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/changes/query.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/changes/schema.py delete mode 100644 src/sqlbuild/compiler/planner/main/changes/_model_changes.py delete mode 100644 src/sqlbuild/compiler/planner/main/changes/relation_replacement.py delete mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_policy.py delete mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_query.py diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index fc478207ea..2a191262b2 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -19,6 +19,7 @@ pub mod lineage; pub mod lineage_command; pub mod lineage_graph; pub mod manifest; +pub mod planner_changes; pub mod python_plan; pub mod python_selection; pub mod python_values; diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/function_hashes.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/function_hashes.rs new file mode 100644 index 0000000000..d3d966b00b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/function_hashes.rs @@ -0,0 +1,47 @@ +//! Function hashes and identity payload equality of fingerprint metadata. + +use serde_json::Value; + +use crate::planner_changes::_helpers::json_payload::{decoded_object, python_equal}; +use crate::planner_changes::constants::LOCAL_FUNCTION_HASHES_METADATA_KEY; +use crate::planner_changes::models::IdentityPayload; + +/// Python equality of two identity payloads. +pub(crate) fn payloads_equal(left: &IdentityPayload, right: &IdentityPayload) -> bool { + match (left, right) { + (IdentityPayload::Object(left), IdentityPayload::Object(right)) => { + python_equal(&Value::Object(left.clone()), &Value::Object(right.clone())) + } + _ => left == right, + } +} + +/// Directly called functions whose recorded hash changed, sorted. +pub(crate) fn changed_local_function_names( + metadata_json: &str, + previous_metadata_json: &str, +) -> Vec { + let previous: Vec<(String, Value)> = local_function_hashes(previous_metadata_json); + let mut changed: Vec = Vec::new(); + for (name, hash) in local_function_hashes(metadata_json) { + let recorded: Option<&Value> = previous + .iter() + .find(|(previous_name, _)| *previous_name == name) + .map(|(_, previous_hash)| previous_hash); + if recorded.is_some_and(|recorded| !python_equal(recorded, &hash)) { + changed.push(name); + } + } + changed.sort(); + changed +} + +fn local_function_hashes(metadata_json: &str) -> Vec<(String, Value)> { + match decoded_object(metadata_json) { + Ok(Some(object)) => match object.get(LOCAL_FUNCTION_HASHES_METADATA_KEY) { + Some(Value::Object(hashes)) => hashes.clone().into_iter().collect(), + _ => Vec::new(), + }, + _ => Vec::new(), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/json_payload.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/json_payload.rs new file mode 100644 index 0000000000..6b7e1ec92b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/json_payload.rs @@ -0,0 +1,80 @@ +//! Decode fingerprint metadata as Python's `json.loads` does and write it as `json.dumps(sort_keys=True, separators=(",", ":"))`. + +use serde_json::{Map, Value}; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; + +/// The decoded top-level object, or None when the text is not a JSON object. +pub(crate) fn decoded_object(metadata_json: &str) -> Result>, ()> { + match serde_json::from_str::(metadata_json) { + Ok(Value::Object(object)) => Ok(Some(object)), + Ok(_) => Ok(None), + Err(_) => Err(()), + } +} + +/// `json.dumps(object, sort_keys=True, separators=(",", ":"), default=str)`. +pub(crate) fn canonical_text(object: Map) -> String { + let dialect: JsonDialect = JsonDialect::Stdlib(StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }); + dumps(&json_value(Value::Object(object)), &dialect).unwrap_or_default() +} + +/// Python equality of two decoded JSON values: numbers by value and booleans as 0 and 1. +pub(crate) fn python_equal(left: &Value, right: &Value) -> bool { + match (left, right) { + (Value::Array(left), Value::Array(right)) => { + left.len() == right.len() + && left + .iter() + .zip(right) + .all(|(left, right)| python_equal(left, right)) + } + (Value::Object(left), Value::Object(right)) => { + left.len() == right.len() + && left.iter().all(|(key, value)| { + right + .get(key) + .is_some_and(|other| python_equal(value, other)) + }) + } + (Value::Number(left_number), Value::Number(right_number)) + if !left_number.is_f64() && !right_number.is_f64() => + { + left_number == right_number + } + (Value::Number(_) | Value::Bool(_), Value::Number(_) | Value::Bool(_)) => { + numeric(left) == numeric(right) + } + _ => left == right, + } +} + +fn numeric(value: &Value) -> Option { + match value { + Value::Bool(flag) => Some(f64::from(u8::from(*flag))), + Value::Number(number) => number.as_f64(), + _ => None, + } +} + +fn json_value(value: Value) -> JsonValue { + match value { + Value::Null => JsonValue::Null, + Value::Bool(flag) => JsonValue::Bool(flag), + Value::Number(number) => match JsonInteger::parse(&number.to_string()) { + Some(integer) if !number.is_f64() => JsonValue::Integer(integer), + _ => JsonValue::Float(number.as_f64().unwrap_or_default()), + }, + Value::String(text) => JsonValue::String(text), + Value::Array(items) => JsonValue::Array(items.into_iter().map(json_value).collect()), + Value::Object(object) => JsonValue::Object( + object + .into_iter() + .map(|(key, value)| (key, json_value(value))) + .collect(), + ), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/mod.rs new file mode 100644 index 0000000000..f344a81462 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod function_hashes; +pub(crate) mod json_payload; +pub(crate) mod pick_more_aggressive; +pub(crate) mod rename_mappings; +pub(crate) mod schema_comparer; +pub(crate) mod with_origin_cursor_input_names; diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/pick_more_aggressive.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/pick_more_aggressive.rs new file mode 100644 index 0000000000..b41082c8ad --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/pick_more_aggressive.rs @@ -0,0 +1,20 @@ +//! Choose between two backfills. + +use crate::planner_changes::models::{Backfill, BackfillAction}; + +/// The more aggressive backfill (full over bounded over forward only); a tie keeps `first`. +pub(crate) fn pick_more_aggressive(first: Backfill, second: Backfill) -> Backfill { + if priority(first.action) >= priority(second.action) { + first + } else { + second + } +} + +fn priority(action: BackfillAction) -> u8 { + match action { + BackfillAction::ForwardOnly => 0, + BackfillAction::Bounded => 1, + BackfillAction::Full => 2, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/rename_mappings.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/rename_mappings.rs new file mode 100644 index 0000000000..b21175a866 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/rename_mappings.rs @@ -0,0 +1,63 @@ +//! Candidate rename mappings and their substituted queries. + +use crate::planner_changes::constants::MAX_CANDIDATE_MAPPINGS; +use crate::planner_changes::models::NameSpan; + +/// The name followed by every origin it was renamed from, breadth first. +pub(crate) fn lineage(name: &str, origins: &[(String, Vec)]) -> Vec { + let mut seen: Vec = vec![name.to_owned()]; + let mut index: usize = 0; + while index < seen.len() { + let current: String = seen[index].clone(); + if let Some((_, list)) = origins.iter().find(|(key, _)| *key == current) { + for origin in list { + if !seen.contains(origin) { + seen.push(origin.clone()); + } + } + } + index += 1; + } + seen +} + +/// `itertools.product(*choices)`, truncated to the first mappings tried. +pub(crate) fn product(choices: &[Vec]) -> Vec> { + let mut combinations: Vec> = vec![Vec::new()]; + for options in choices { + let mut next: Vec> = Vec::new(); + for prefix in &combinations { + for option in options { + if next.len() == MAX_CANDIDATE_MAPPINGS { + break; + } + let mut combination: Vec = prefix.clone(); + combination.push(option.clone()); + next.push(combination); + } + } + combinations = next; + } + combinations +} + +/// The query with each span replaced by its mapped origin name. +pub(crate) fn substitute( + characters: &[char], + spans: &[NameSpan], + mapping: &[(String, String)], +) -> String { + let mut text: String = String::new(); + let mut copied_to: usize = 0; + for span in spans { + text.extend(&characters[copied_to.min(span.start)..span.start]); + let replacement: &str = mapping + .iter() + .find(|(name, _)| *name == span.name) + .map_or(span.name.as_str(), |(_, origin)| origin.as_str()); + text.push_str(replacement); + copied_to = span.end; + } + text.extend(&characters[copied_to.min(characters.len())..]); + text +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/schema_comparer.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/schema_comparer.rs new file mode 100644 index 0000000000..0f5cff7051 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/schema_comparer.rs @@ -0,0 +1,159 @@ +//! Column-by-column schema comparison state. + +use std::collections::HashSet; + +use crate::planner_changes::models::{ + BackfillFacts, SchemaChangeKind, SchemaColumnSource, SchemaColumns, SchemaFindingRow, +}; + +/// Whether schema findings replay the policy: only alongside a definition, config or declared-column change. +pub(crate) fn schema_replays(facts: BackfillFacts) -> bool { + facts.schema_changed + && (facts.query_changed + || facts.functions_changed + || facts.config_changed + || facts.declared_columns_changed) +} + +/// Expected columns with optional types, and warehouse columns with dictionary semantics. +pub(crate) struct FoldedColumns { + pub(crate) yml: Vec<(String, Option)>, + pub(crate) inferred: Option)>>, + pub(crate) warehouse: Vec<(String, String)>, +} + +/// Lower-case names when the dialect folds them; later duplicate warehouse names replace earlier types in place. +pub(crate) fn folded_columns(columns: SchemaColumns, case_folding: bool) -> FoldedColumns { + let fold = |name: String| { + if case_folding { + name.to_lowercase() + } else { + name + } + }; + let yml: Vec<(String, Option)> = columns + .yml + .into_iter() + .map(|(name, kind)| (fold(name), Some(kind))) + .collect(); + let inferred: Option)>> = columns.inferred.map(|items| { + let mut folded: Vec<(String, Option)> = Vec::with_capacity(items.len()); + for (name, kind) in items { + folded.push((fold(name), kind)); + } + folded + }); + let mut warehouse: Vec<(String, String)> = Vec::new(); + for (name, kind) in columns.warehouse { + let name: String = fold(name); + match warehouse.iter_mut().find(|(existing, _)| *existing == name) { + Some(entry) => entry.1 = kind, + None => warehouse.push((name, kind)), + } + } + FoldedColumns { + yml, + inferred, + warehouse, + } +} + +/// Findings collected so far and the expected names already compared. +pub(crate) struct Comparer<'a, 'b, E> { + pub(crate) warehouse: &'a [(String, String)], + pub(crate) seen: HashSet, + pub(crate) findings: Vec, + pub(crate) types_equal: &'b mut dyn FnMut(&str, &str) -> Result, +} + +impl Comparer<'_, '_, E> { + /// Added and retyped columns; `skip_seen` skips names an earlier source already covered; then marks them seen. + pub(crate) fn compare( + &mut self, + expected: &[(String, Option)], + source: SchemaColumnSource, + skip_seen: bool, + ) -> Result<(), E> { + for (name, kind) in expected { + if skip_seen && self.seen.contains(name) { + continue; + } + match self.warehouse.iter().find(|(existing, _)| existing == name) { + None => self.findings.push(finding( + (SchemaChangeKind::ColumnAdded, name, source), + kind.clone(), + None, + )), + Some((_, actual)) => { + if let Some(expected_type) = kind + && !(self.types_equal)(actual, expected_type)? + { + self.findings.push(finding( + (SchemaChangeKind::ColumnTypeChanged, name, source), + Some(expected_type.clone()), + Some(actual.clone()), + )); + } + } + } + } + self.seen + .extend(expected.iter().map(|(name, _)| name.clone())); + Ok(()) + } + + /// Retyped warehouse columns that exactly one dynamic family accepts. + pub(crate) fn compare_families( + &mut self, + single_family_type: &mut dyn FnMut(&str) -> Result, E>, + ) -> Result<(), E> { + for (name, kind) in self.warehouse { + if self.seen.contains(name) { + continue; + } + let Some(family_type) = single_family_type(name)? else { + continue; + }; + self.seen.insert(name.clone()); + if !(self.types_equal)(&family_type, kind)? { + self.findings.push(finding( + ( + SchemaChangeKind::ColumnTypeChanged, + name, + SchemaColumnSource::Yml, + ), + Some(family_type), + Some(kind.clone()), + )); + } + } + Ok(()) + } + + /// Warehouse columns no expected column or family covered. + pub(crate) fn report_removed(&mut self, source: SchemaColumnSource) { + for (name, kind) in self.warehouse { + if !self.seen.contains(name) { + self.findings.push(finding( + (SchemaChangeKind::ColumnRemoved, name, source), + None, + Some(kind.clone()), + )); + } + } + } +} + +fn finding( + (kind, name, source): (SchemaChangeKind, &str, SchemaColumnSource), + expected_type: Option, + actual_type: Option, +) -> SchemaFindingRow { + SchemaFindingRow { + kind, + column_name: name.to_owned(), + source, + expected_type, + actual_type, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/with_origin_cursor_input_names.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/with_origin_cursor_input_names.rs new file mode 100644 index 0000000000..77bd0cc2bc --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/with_origin_cursor_input_names.rs @@ -0,0 +1,37 @@ +//! Name renamed cursor inputs by their origin models. + +use serde_json::{Map, Value}; + +use crate::planner_changes::_helpers::json_payload::{canonical_text, decoded_object}; +use crate::planner_changes::constants::{CONFIG_METADATA_KEY, CURSOR_INPUTS_CONFIG_KEY}; + +/// Canonical metadata JSON whose `config.cursor_inputs` keys use origin names; other text is returned unchanged. +pub(crate) fn with_origin_cursor_input_names( + metadata_json: &str, + renamed_refs: &[(String, String)], +) -> String { + let Ok(Some(mut object)) = decoded_object(metadata_json) else { + return metadata_json.to_owned(); + }; + let Some(Value::Object(config)) = object.get(CONFIG_METADATA_KEY) else { + return metadata_json.to_owned(); + }; + let Some(Value::Object(cursor_inputs)) = config.get(CURSOR_INPUTS_CONFIG_KEY) else { + return metadata_json.to_owned(); + }; + let mut origin_inputs: Map = Map::new(); + for (name, value) in cursor_inputs { + let origin: &str = renamed_refs + .iter() + .find(|(renamed, _)| renamed == name) + .map_or(name.as_str(), |(_, origin)| origin.as_str()); + origin_inputs.insert(origin.to_owned(), value.clone()); + } + let mut config: Map = config.clone(); + config.insert( + CURSOR_INPUTS_CONFIG_KEY.to_owned(), + Value::Object(origin_inputs), + ); + object.insert(CONFIG_METADATA_KEY.to_owned(), Value::Object(config)); + canonical_text(object) +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/constants.rs b/crates/sqlbuild-analysis/src/planner_changes/constants.rs new file mode 100644 index 0000000000..0ff28f52b8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/constants.rs @@ -0,0 +1,26 @@ +//! Planner change-detection constants shared with the Python planner. + +/// Fingerprint metadata key of the recorded audit gate. +pub const AUDIT_GATE_METADATA_KEY: &str = "audit_gate"; +/// Fingerprint metadata key of the migration fingerprint. +pub const MIGRATION_FINGERPRINT_METADATA_KEY: &str = "migration_fingerprint"; +/// Fingerprint metadata key of the declared-columns hash. +pub const DECLARED_COLUMNS_METADATA_KEY: &str = "declared_columns_hash"; +/// Fingerprint metadata key of directly called function hashes. +pub const LOCAL_FUNCTION_HASHES_METADATA_KEY: &str = "local_function_hashes"; +/// Fingerprint metadata key of the model config. +pub const CONFIG_METADATA_KEY: &str = "config"; +/// Model config key of cursor inputs. +pub const CURSOR_INPUTS_CONFIG_KEY: &str = "cursor_inputs"; +/// Metadata a fingerprint row without metadata records. +pub const EMPTY_FINGERPRINT_METADATA_JSON: &str = "{}"; +/// The replay policy value that replays nothing. +pub const FORWARD_ONLY_POLICY: &str = "forward"; +/// The replay policy value that replays everything. +pub const FULL_POLICY: &str = "full"; +/// Prefix of a bounded replay policy. +pub const BOUNDED_POLICY_PREFIX: &str = "bounded-"; +/// The dialect whose unquoted column names fold to lower case for comparison. +pub const CASE_FOLDING_DIALECT: &str = "snowflake"; +/// At most this many rename mappings are tried for one query. +pub const MAX_CANDIDATE_MAPPINGS: usize = 64; diff --git a/crates/sqlbuild-analysis/src/planner_changes/errors.rs b/crates/sqlbuild-analysis/src/planner_changes/errors.rs new file mode 100644 index 0000000000..7ba2438e8f --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/errors.rs @@ -0,0 +1,12 @@ +//! Planner change-detection errors. + +/// A `replay_on_change` value that is not a known policy. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct UnknownReplayPolicy(pub String); + +impl UnknownReplayPolicy { + /// The planner input error message. + pub fn message(&self) -> String { + format!("unknown replay_on_change '{}'", self.0) + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/detect_schema_changes.rs b/crates/sqlbuild-analysis/src/planner_changes/main/detect_schema_changes.rs new file mode 100644 index 0000000000..a0c52471c3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/detect_schema_changes.rs @@ -0,0 +1,45 @@ +//! Compare declared and inferred columns with the warehouse relation's columns. + +use std::collections::HashSet; + +use crate::planner_changes::_helpers::schema_comparer::{Comparer, FoldedColumns, folded_columns}; +use crate::planner_changes::models::{ + SchemaColumnSource, SchemaColumns, SchemaComparison, SchemaFindingRow, +}; +use crate::planner_changes::types::SchemaChecks; + +/// Findings in Python's order: declared and inferred columns, dynamic families, then removed columns. +pub fn detect_schema_changes( + columns: SchemaColumns, + comparison: SchemaComparison, + (types_equal, single_family_type): SchemaChecks<'_, E>, +) -> Result, E> { + let FoldedColumns { + yml, + inferred, + warehouse, + } = folded_columns(columns, comparison.case_folding); + let mut comparer = Comparer { + warehouse: &warehouse, + seen: HashSet::new(), + findings: Vec::new(), + types_equal, + }; + let inferred: &[(String, Option)] = inferred.as_deref().unwrap_or_default(); + if comparison.type_enforcement { + comparer.compare(&yml, SchemaColumnSource::Yml, false)?; + comparer.compare(inferred, SchemaColumnSource::SqlAnalysis, true)?; + } else { + comparer.compare(inferred, SchemaColumnSource::SqlAnalysis, true)?; + comparer.compare(&yml, SchemaColumnSource::Yml, true)?; + } + comparer.compare_families(single_family_type)?; + if comparison.type_enforcement || comparison.inferred_schema_complete { + comparer.report_removed(if yml.is_empty() { + SchemaColumnSource::SqlAnalysis + } else { + SchemaColumnSource::Yml + }); + } + Ok(comparer.findings) +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/metadata_changes.rs b/crates/sqlbuild-analysis/src/planner_changes/main/metadata_changes.rs new file mode 100644 index 0000000000..cd6a4ef57b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/metadata_changes.rs @@ -0,0 +1,37 @@ +//! Compare current and recorded version-identity metadata. + +use crate::planner_changes::_helpers::function_hashes::{ + changed_local_function_names, payloads_equal, +}; +use crate::planner_changes::_helpers::with_origin_cursor_input_names::with_origin_cursor_input_names; +use crate::planner_changes::main::version_identity_payload::version_identity_payload; +use crate::planner_changes::models::MetadataChanges; + +/// Called functions whose recorded hash changed (when queries are tracked) and whether the remaining identity changed. +pub fn metadata_changes( + (metadata_json, recorded_metadata_json): (&str, Option<&str>), + origin_names: Option<&[(String, String)]>, + query_change_tracking: bool, +) -> MetadataChanges { + let Some(recorded) = recorded_metadata_json else { + return MetadataChanges::default(); + }; + let changed_functions: Vec = if query_change_tracking { + changed_local_function_names(metadata_json, recorded) + } else { + Vec::new() + }; + let current: String = match origin_names { + Some(names) => with_origin_cursor_input_names(metadata_json, names), + None => metadata_json.to_owned(), + }; + let without_functions: bool = !changed_functions.is_empty(); + let config_changed: bool = !payloads_equal( + &version_identity_payload(Some(¤t), without_functions), + &version_identity_payload(Some(recorded), without_functions), + ); + MetadataChanges { + changed_functions, + config_changed, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/mod.rs b/crates/sqlbuild-analysis/src/planner_changes/main/mod.rs new file mode 100644 index 0000000000..86949d0c09 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/mod.rs @@ -0,0 +1,11 @@ +pub mod detect_schema_changes; +pub mod metadata_changes; +pub mod model_backfill; +pub mod model_change_kind; +pub mod origin_reference_names; +pub mod recorded_declared_columns_hash; +pub mod replay_policy_needed; +pub mod resolve_replay_on_change; +pub mod version_identity_payload; +pub mod with_declared_columns_hash; +pub mod without_declared_columns_hash; diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/model_backfill.rs b/crates/sqlbuild-analysis/src/planner_changes/main/model_backfill.rs new file mode 100644 index 0000000000..96878d4832 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/model_backfill.rs @@ -0,0 +1,27 @@ +//! Resolve the backfill of a detected change. + +use crate::planner_changes::_helpers::pick_more_aggressive::pick_more_aggressive; +use crate::planner_changes::_helpers::schema_comparer::schema_replays; +use crate::planner_changes::models::{Backfill, BackfillAction, BackfillFacts}; + +/// The replay policy applies to query or function changes, and to schema findings that come with any definition change; `replay` is the resolved policy, present when `replay_policy_needed`. +pub fn model_backfill(facts: BackfillFacts, replay: Option) -> Backfill { + let forward = Backfill { + action: BackfillAction::ForwardOnly, + duration: None, + }; + let Some(replay) = replay else { + return forward; + }; + let definition: Backfill = if facts.query_changed || facts.functions_changed { + replay.clone() + } else { + forward.clone() + }; + let schema: Backfill = if schema_replays(facts) { + replay + } else { + forward + }; + pick_more_aggressive(definition, schema) +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/model_change_kind.rs b/crates/sqlbuild-analysis/src/planner_changes/main/model_change_kind.rs new file mode 100644 index 0000000000..9e8f5cf00c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/model_change_kind.rs @@ -0,0 +1,18 @@ +//! Classify a model's change. + +use crate::planner_changes::models::{BackfillFacts, ChangeKind}; + +/// The change kind by precedence: query, then called functions, schema, config, else no change. +pub fn model_change_kind(facts: BackfillFacts) -> ChangeKind { + if facts.query_changed { + ChangeKind::QueryChanged + } else if facts.functions_changed { + ChangeKind::FunctionChanged + } else if facts.schema_changed { + ChangeKind::SchemaChanged + } else if facts.config_changed { + ChangeKind::ConfigChanged + } else { + ChangeKind::NoChange + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/origin_reference_names.rs b/crates/sqlbuild-analysis/src/planner_changes/main/origin_reference_names.rs new file mode 100644 index 0000000000..ab53e6e1ef --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/origin_reference_names.rs @@ -0,0 +1,54 @@ +//! Recognize a query whose only change is `__ref` names rewritten for renamed models. + +use std::collections::BTreeSet; + +use crate::planner_changes::_helpers::rename_mappings::{lineage, product, substitute}; +use crate::planner_changes::constants::MAX_CANDIDATE_MAPPINGS; +use crate::planner_changes::models::{NameSpan, RenameCandidate}; +use crate::planner_changes::types::RenameMatching; + +/// The first rename mapping (in name order, at most 64 tried) under which the query hashes to `recorded_hash`. +pub fn origin_reference_names( + (query_sql, recorded_hash): (&str, &str), + renames: &[RenameCandidate], + (find_spans, query_hash): RenameMatching<'_, E>, +) -> Result>, E> { + let mut origins: Vec<(String, Vec)> = Vec::new(); + for rename in renames.iter().filter(|rename| rename.applies) { + match origins + .iter_mut() + .find(|(name, _)| *name == rename.new_name) + { + Some((_, list)) => list.push(rename.origin_name.clone()), + None => origins.push((rename.new_name.clone(), vec![rename.origin_name.clone()])), + } + } + if origins.is_empty() { + return Ok(None); + } + let keys: BTreeSet = origins.iter().map(|(name, _)| name.clone()).collect(); + let spans: Vec = find_spans(&keys)?; + let names: Vec = spans + .iter() + .map(|span| span.name.clone()) + .collect::>() + .into_iter() + .collect(); + let choices: Vec> = names.iter().map(|name| lineage(name, &origins)).collect(); + let characters: Vec = query_sql.chars().collect(); + for combination in product(&choices).into_iter().take(MAX_CANDIDATE_MAPPINGS) { + let mapping: Vec<(String, String)> = names + .iter() + .zip(combination) + .filter(|(name, origin)| *name != origin) + .map(|(name, origin)| (name.clone(), origin)) + .collect(); + if mapping.is_empty() { + continue; + } + if query_hash(&substitute(&characters, &spans, &mapping))? == recorded_hash { + return Ok(Some(mapping)); + } + } + Ok(None) +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/recorded_declared_columns_hash.rs b/crates/sqlbuild-analysis/src/planner_changes/main/recorded_declared_columns_hash.rs new file mode 100644 index 0000000000..6a3fa0c963 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/recorded_declared_columns_hash.rs @@ -0,0 +1,17 @@ +//! Read the declared-columns hash a fingerprint recorded. + +use serde_json::Value; + +use crate::planner_changes::_helpers::json_payload::decoded_object; +use crate::planner_changes::constants::DECLARED_COLUMNS_METADATA_KEY; + +/// The recorded declared-columns hash, when the metadata holds one as a string. +pub fn recorded_declared_columns_hash(metadata_json: Option<&str>) -> Option { + match decoded_object(metadata_json?) { + Ok(Some(object)) => match object.get(DECLARED_COLUMNS_METADATA_KEY) { + Some(Value::String(hash)) => Some(hash.clone()), + _ => None, + }, + _ => None, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/replay_policy_needed.rs b/crates/sqlbuild-analysis/src/planner_changes/main/replay_policy_needed.rs new file mode 100644 index 0000000000..22471ad68c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/replay_policy_needed.rs @@ -0,0 +1,9 @@ +//! Whether a change reads the model's replay policy. + +use crate::planner_changes::_helpers::schema_comparer::schema_replays; +use crate::planner_changes::models::BackfillFacts; + +/// Query and function changes read `replay_on_change`, and so do schema findings that come with a definition change. +pub fn replay_policy_needed(facts: BackfillFacts) -> bool { + facts.query_changed || facts.functions_changed || schema_replays(facts) +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/resolve_replay_on_change.rs b/crates/sqlbuild-analysis/src/planner_changes/main/resolve_replay_on_change.rs new file mode 100644 index 0000000000..90ed3bf650 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/resolve_replay_on_change.rs @@ -0,0 +1,24 @@ +//! Parse a model's `replay_on_change` policy. + +use crate::planner_changes::constants::{BOUNDED_POLICY_PREFIX, FORWARD_ONLY_POLICY, FULL_POLICY}; +use crate::planner_changes::errors::UnknownReplayPolicy; +use crate::planner_changes::models::{Backfill, BackfillAction}; + +/// The backfill a detected version change replays: forward only, full, or bounded by a duration. +pub fn resolve_replay_on_change(raw: Option<&str>) -> Result { + let action = |action: BackfillAction, duration: Option| Backfill { action, duration }; + match raw { + None | Some(FORWARD_ONLY_POLICY) => Ok(action(BackfillAction::ForwardOnly, None)), + Some(FULL_POLICY) => Ok(action(BackfillAction::Full, None)), + Some(value) => match value + .strip_prefix(BOUNDED_POLICY_PREFIX) + .map(|duration| duration.strip_suffix('\n').unwrap_or(duration)) + { + Some(duration) if !duration.is_empty() && !duration.contains('\n') => Ok(action( + BackfillAction::Bounded, + Some(duration.trim().to_owned()), + )), + _ => Err(UnknownReplayPolicy(value.to_owned())), + }, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs b/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs new file mode 100644 index 0000000000..9bceac24a7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs @@ -0,0 +1,35 @@ +//! The fingerprint metadata that participates in version identity. + +use crate::planner_changes::_helpers::json_payload::decoded_object; +use crate::planner_changes::constants::{ + AUDIT_GATE_METADATA_KEY, DECLARED_COLUMNS_METADATA_KEY, LOCAL_FUNCTION_HASHES_METADATA_KEY, + MIGRATION_FINGERPRINT_METADATA_KEY, +}; +use crate::planner_changes::models::IdentityPayload; + +/// The metadata without audit-gate, migration and declared-column entries; with `without_functions`, also without function hashes. +pub fn version_identity_payload( + metadata_json: Option<&str>, + without_functions: bool, +) -> IdentityPayload { + let Some(text) = metadata_json else { + return IdentityPayload::Absent; + }; + match decoded_object(text) { + Err(()) => IdentityPayload::Raw(text.to_owned()), + Ok(None) => IdentityPayload::Absent, + Ok(Some(mut object)) => { + for key in [ + AUDIT_GATE_METADATA_KEY, + MIGRATION_FINGERPRINT_METADATA_KEY, + DECLARED_COLUMNS_METADATA_KEY, + ] { + object.shift_remove(key); + } + if without_functions { + object.shift_remove(LOCAL_FUNCTION_HASHES_METADATA_KEY); + } + IdentityPayload::Object(object) + } + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/with_declared_columns_hash.rs b/crates/sqlbuild-analysis/src/planner_changes/main/with_declared_columns_hash.rs new file mode 100644 index 0000000000..c3508647a9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/with_declared_columns_hash.rs @@ -0,0 +1,20 @@ +//! Record the declared-columns hash in fingerprint metadata. + +use serde_json::Value; + +use crate::planner_changes::_helpers::json_payload::{canonical_text, decoded_object}; +use crate::planner_changes::constants::DECLARED_COLUMNS_METADATA_KEY; + +/// Canonical metadata JSON that also records `declared_columns_hash`; text that is not an object is returned unchanged. +pub fn with_declared_columns_hash(metadata_json: &str, declared_columns_hash: &str) -> String { + match decoded_object(metadata_json) { + Ok(Some(mut object)) => { + object.insert( + DECLARED_COLUMNS_METADATA_KEY.to_owned(), + Value::String(declared_columns_hash.to_owned()), + ); + canonical_text(object) + } + _ => metadata_json.to_owned(), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/without_declared_columns_hash.rs b/crates/sqlbuild-analysis/src/planner_changes/main/without_declared_columns_hash.rs new file mode 100644 index 0000000000..a01da9c91e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/main/without_declared_columns_hash.rs @@ -0,0 +1,15 @@ +//! Drop the declared-columns hash from fingerprint metadata. + +use crate::planner_changes::_helpers::json_payload::{canonical_text, decoded_object}; +use crate::planner_changes::constants::DECLARED_COLUMNS_METADATA_KEY; + +/// Canonical metadata JSON without the declared-columns hash, or the text unchanged when it has none. +pub fn without_declared_columns_hash(metadata_json: &str) -> String { + match decoded_object(metadata_json) { + Ok(Some(mut object)) if object.contains_key(DECLARED_COLUMNS_METADATA_KEY) => { + object.shift_remove(DECLARED_COLUMNS_METADATA_KEY); + canonical_text(object) + } + _ => metadata_json.to_owned(), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/mod.rs b/crates/sqlbuild-analysis/src/planner_changes/mod.rs new file mode 100644 index 0000000000..9afa04ee22 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/mod.rs @@ -0,0 +1,11 @@ +//! Planner change detection: replay policy, version-identity metadata, schema findings, reference-rename origins and the per-model change decision. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/planner_changes/models.rs b/crates/sqlbuild-analysis/src/planner_changes/models.rs new file mode 100644 index 0000000000..71d7e90a84 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/models.rs @@ -0,0 +1,108 @@ +//! Planner change-detection inputs and results. + +/// The replay a change asks for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BackfillAction { + Full, + Bounded, + ForwardOnly, +} + +/// A backfill action with the duration of a bounded replay. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Backfill { + pub action: BackfillAction, + pub duration: Option, +} + +/// How a model changed since its recorded build, in precedence order. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ChangeKind { + QueryChanged, + FunctionChanged, + SchemaChanged, + ConfigChanged, + NoChange, +} + +/// What a schema finding reports about one column. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SchemaChangeKind { + ColumnAdded, + ColumnRemoved, + ColumnTypeChanged, +} + +/// Where an expected column was declared. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SchemaColumnSource { + Yml, + SqlAnalysis, +} + +/// One difference between expected and warehouse columns. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SchemaFindingRow { + pub kind: SchemaChangeKind, + pub column_name: String, + pub source: SchemaColumnSource, + pub expected_type: Option, + pub actual_type: Option, +} + +/// Expected and warehouse columns of one model: name and type. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SchemaColumns { + pub yml: Vec<(String, String)>, + pub inferred: Option)>>, + pub warehouse: Vec<(String, String)>, +} + +/// How a model's columns are compared. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct SchemaComparison { + pub type_enforcement: bool, + pub inferred_schema_complete: bool, + pub case_folding: bool, +} + +/// The parts of version-identity metadata that changed since the recorded build. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct MetadataChanges { + pub changed_functions: Vec, + pub config_changed: bool, +} + +/// Facts the backfill of a detected change depends on. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct BackfillFacts { + pub query_changed: bool, + pub functions_changed: bool, + pub config_changed: bool, + pub declared_columns_changed: bool, + pub schema_changed: bool, +} + +/// One model rename a reference may have been rewritten for; `applies` when the recorded build predates it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RenameCandidate { + pub new_name: String, + pub origin_name: String, + pub applies: bool, +} + +/// A `__ref` name in the query: start and end character offsets and the name. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NameSpan { + pub start: usize, + pub end: usize, + pub name: String, +} + +/// Fingerprint metadata as version identity compares it: absent, undecodable text, or the decoded object. +#[derive(Clone, Debug, PartialEq)] +pub enum IdentityPayload { + Absent, + Raw(String), + Object(serde_json::Map), +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/tests.rs b/crates/sqlbuild-analysis/src/planner_changes/tests.rs new file mode 100644 index 0000000000..949a25ebbd --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_planner_changes.rs"] +mod planner_changes; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_changes/tests/helpers.rs b/crates/sqlbuild-analysis/src/planner_changes/tests/helpers.rs new file mode 100644 index 0000000000..2412088ca4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/tests/helpers.rs @@ -0,0 +1,46 @@ +use crate::planner_changes::models::{ + Backfill, BackfillAction, NameSpan, SchemaChangeKind, SchemaColumnSource, SchemaFindingRow, +}; + +pub(super) fn backfill(action: BackfillAction, duration: Option<&str>) -> Backfill { + Backfill { + action, + duration: duration.map(str::to_owned), + } +} + +pub(super) fn finding( + (kind, name, source): (SchemaChangeKind, &str, SchemaColumnSource), + expected_type: Option<&str>, + actual_type: Option<&str>, +) -> SchemaFindingRow { + SchemaFindingRow { + kind, + column_name: name.to_owned(), + source, + expected_type: expected_type.map(str::to_owned), + actual_type: actual_type.map(str::to_owned), + } +} + +pub(super) fn pairs(items: &[(&str, &str)]) -> Vec<(String, String)> { + items + .iter() + .map(|(left, right)| ((*left).to_owned(), (*right).to_owned())) + .collect() +} + +/// Case-insensitive type equality, as a stand-in for the adapter's type normalization. +pub(super) fn types_equal(left: &str, right: &str) -> Result { + Ok(left.eq_ignore_ascii_case(right)) +} + +/// The span of the one quoted `name` in an ASCII query. +pub(super) fn quoted_span(query_sql: &str, name: &str) -> NameSpan { + let start: usize = query_sql.find(&format!("\"{name}\"")).unwrap_or_default() + 1; + NameSpan { + start, + end: start + name.len(), + name: name.to_owned(), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/tests/test_planner_changes.rs b/crates/sqlbuild-analysis/src/planner_changes/tests/test_planner_changes.rs new file mode 100644 index 0000000000..6e82bec30e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/tests/test_planner_changes.rs @@ -0,0 +1,427 @@ +use crate::planner_changes::main::detect_schema_changes::detect_schema_changes; +use crate::planner_changes::main::metadata_changes::metadata_changes; +use crate::planner_changes::main::model_backfill::model_backfill; +use crate::planner_changes::main::model_change_kind::model_change_kind; +use crate::planner_changes::main::origin_reference_names::origin_reference_names; +use crate::planner_changes::main::recorded_declared_columns_hash::recorded_declared_columns_hash; +use crate::planner_changes::main::replay_policy_needed::replay_policy_needed; +use crate::planner_changes::main::resolve_replay_on_change::resolve_replay_on_change; +use crate::planner_changes::main::with_declared_columns_hash::with_declared_columns_hash; +use crate::planner_changes::main::without_declared_columns_hash::without_declared_columns_hash; +use crate::planner_changes::models::{ + BackfillAction, BackfillFacts, ChangeKind, NameSpan, RenameCandidate, SchemaChangeKind, + SchemaColumnSource, SchemaColumns, SchemaComparison, +}; +use crate::planner_changes::tests::helpers::{backfill, finding, pairs, quoted_span, types_equal}; +use crate::planner_changes::tests::test_types::{ + DecisionTestCase, MetadataChangesTestCase, MetadataTextTestCase, OriginTestCase, + ReplayTestCase, SchemaTestCase, +}; + +#[test] +fn given_replay_policy_when_resolving_then_matches_python_parsing() { + let test_cases = [ + ReplayTestCase { + description: "absent", + raw: None, + expected_backfill: Ok(backfill(BackfillAction::ForwardOnly, None)), + }, + ReplayTestCase { + description: "forward", + raw: Some("forward"), + expected_backfill: Ok(backfill(BackfillAction::ForwardOnly, None)), + }, + ReplayTestCase { + description: "full", + raw: Some("full"), + expected_backfill: Ok(backfill(BackfillAction::Full, None)), + }, + ReplayTestCase { + description: "bounded with spaces", + raw: Some("bounded- 7d "), + expected_backfill: Ok(backfill(BackfillAction::Bounded, Some("7d"))), + }, + ReplayTestCase { + description: "bounded before a final newline", + raw: Some("bounded-3d\n"), + expected_backfill: Ok(backfill(BackfillAction::Bounded, Some("3d"))), + }, + ReplayTestCase { + description: "unknown", + raw: Some("forward_only"), + expected_backfill: Err("unknown replay_on_change 'forward_only'".to_owned()), + }, + ReplayTestCase { + description: "bounded without a duration", + raw: Some("bounded-"), + expected_backfill: Err("unknown replay_on_change 'bounded-'".to_owned()), + }, + ]; + for test_case in test_cases { + assert_eq!( + resolve_replay_on_change(test_case.raw).map_err(|error| error.message()), + test_case.expected_backfill, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_metadata_json_when_recording_declared_columns_then_writes_python_json() { + let test_cases = [ + MetadataTextTestCase { + description: "unsorted object with unicode and numbers", + metadata_json: r#"{"model_name": "orders", "config": {"b": 1.0, "a": "café"}, "n": 10}"#, + expected_with_hash: r#"{"config":{"a":"caf\u00e9","b":1.0},"declared_columns_hash":"h1","model_name":"orders","n":10}"#, + expected_without_hash: r#"{"model_name": "orders", "config": {"b": 1.0, "a": "café"}, "n": 10}"#, + expected_recorded_hash: None, + }, + MetadataTextTestCase { + description: "existing hash", + metadata_json: r#"{"declared_columns_hash":"h0","x":[1,2]}"#, + expected_with_hash: r#"{"declared_columns_hash":"h1","x":[1,2]}"#, + expected_without_hash: r#"{"x":[1,2]}"#, + expected_recorded_hash: Some("h0"), + }, + MetadataTextTestCase { + description: "not an object", + metadata_json: "[1]", + expected_with_hash: "[1]", + expected_without_hash: "[1]", + expected_recorded_hash: None, + }, + MetadataTextTestCase { + description: "not JSON", + metadata_json: "{oops", + expected_with_hash: "{oops", + expected_without_hash: "{oops", + expected_recorded_hash: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + ( + with_declared_columns_hash(test_case.metadata_json, "h1"), + without_declared_columns_hash(test_case.metadata_json), + recorded_declared_columns_hash(Some(test_case.metadata_json)), + ), + ( + test_case.expected_with_hash.to_owned(), + test_case.expected_without_hash.to_owned(), + test_case.expected_recorded_hash.map(str::to_owned), + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_recorded_metadata_when_comparing_then_reports_function_and_config_changes() { + let test_cases = [ + MetadataChangesTestCase { + description: "no recorded metadata", + current: r#"{"config":{"materialized":"table"}}"#, + recorded: None, + origin_names: None, + expected_functions: Vec::new(), + expected_config_changed: false, + }, + MetadataChangesTestCase { + description: "audit gate and declared columns are ignored, numbers compare by value", + current: r#"{"config":{"batch":1},"audit_gate":{"a":1},"declared_columns_hash":"x"}"#, + recorded: Some(r#"{"config":{"batch":1.0},"declared_columns_hash":"y"}"#), + origin_names: None, + expected_functions: Vec::new(), + expected_config_changed: false, + }, + MetadataChangesTestCase { + description: "config change", + current: r#"{"config":{"materialized":"view"}}"#, + recorded: Some(r#"{"config":{"materialized":"table"}}"#), + origin_names: None, + expected_functions: Vec::new(), + expected_config_changed: true, + }, + MetadataChangesTestCase { + description: "only function hashes changed", + current: r#"{"config":{},"local_function_hashes":{"zeta":"2","alpha":"2","new":"1"}}"#, + recorded: Some(r#"{"config":{},"local_function_hashes":{"zeta":"1","alpha":"1"}}"#), + origin_names: None, + expected_functions: vec!["alpha".to_owned(), "zeta".to_owned()], + expected_config_changed: false, + }, + MetadataChangesTestCase { + description: "renamed cursor input named by its origin", + current: r#"{"config":{"cursor_inputs":{"orders_v2":"ordered_at"}}}"#, + recorded: Some(r#"{"config":{"cursor_inputs":{"orders":"ordered_at"}}}"#), + origin_names: Some(pairs(&[("orders_v2", "orders")])), + expected_functions: Vec::new(), + expected_config_changed: false, + }, + ]; + for test_case in test_cases { + let changes = metadata_changes( + (test_case.current, test_case.recorded), + test_case.origin_names.as_deref(), + true, + ); + assert_eq!( + (changes.changed_functions, changes.config_changed), + ( + test_case.expected_functions, + test_case.expected_config_changed + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_change_facts_when_deciding_then_kind_and_backfill_follow_precedence() { + let test_cases = [ + DecisionTestCase { + description: "query change replays the policy", + facts: BackfillFacts { + query_changed: true, + schema_changed: true, + ..BackfillFacts::default() + }, + replay_on_change: Some("bounded-2d"), + expected_kind: ChangeKind::QueryChanged, + expected_backfill: backfill(BackfillAction::Bounded, Some("2d")), + }, + DecisionTestCase { + description: "schema drift alone stays forward", + facts: BackfillFacts { + schema_changed: true, + config_changed: false, + ..BackfillFacts::default() + }, + replay_on_change: Some("full"), + expected_kind: ChangeKind::SchemaChanged, + expected_backfill: backfill(BackfillAction::ForwardOnly, None), + }, + DecisionTestCase { + description: "schema findings with a config change replay", + facts: BackfillFacts { + schema_changed: true, + config_changed: true, + ..BackfillFacts::default() + }, + replay_on_change: Some("full"), + expected_kind: ChangeKind::SchemaChanged, + expected_backfill: backfill(BackfillAction::Full, None), + }, + DecisionTestCase { + description: "config change alone", + facts: BackfillFacts { + config_changed: true, + ..BackfillFacts::default() + }, + replay_on_change: Some("full"), + expected_kind: ChangeKind::ConfigChanged, + expected_backfill: backfill(BackfillAction::ForwardOnly, None), + }, + DecisionTestCase { + description: "function change", + facts: BackfillFacts { + functions_changed: true, + ..BackfillFacts::default() + }, + replay_on_change: None, + expected_kind: ChangeKind::FunctionChanged, + expected_backfill: backfill(BackfillAction::ForwardOnly, None), + }, + ]; + for test_case in test_cases { + assert_eq!( + ( + model_change_kind(test_case.facts), + model_backfill( + test_case.facts, + replay_policy_needed(test_case.facts) + .then(|| resolve_replay_on_change(test_case.replay_on_change).ok()) + .flatten(), + ) + ), + (test_case.expected_kind, test_case.expected_backfill), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_columns_when_comparing_schema_then_findings_keep_python_order() { + let test_cases = [ + SchemaTestCase { + description: "enforced declared columns, inferred extras and a removed column", + columns: SchemaColumns { + yml: pairs(&[("id", "INTEGER"), ("label", "TEXT")]), + inferred: Some(vec![ + ("id".to_owned(), Some("BIGINT".to_owned())), + ("amount".to_owned(), None), + ]), + warehouse: pairs(&[("id", "integer"), ("label", "INTEGER"), ("old", "TEXT")]), + }, + comparison: SchemaComparison { + type_enforcement: true, + inferred_schema_complete: false, + case_folding: false, + }, + expected_findings: vec![ + finding( + ( + SchemaChangeKind::ColumnTypeChanged, + "label", + SchemaColumnSource::Yml, + ), + Some("TEXT"), + Some("INTEGER"), + ), + finding( + ( + SchemaChangeKind::ColumnAdded, + "amount", + SchemaColumnSource::SqlAnalysis, + ), + None, + None, + ), + finding( + ( + SchemaChangeKind::ColumnRemoved, + "old", + SchemaColumnSource::Yml, + ), + None, + Some("TEXT"), + ), + ], + }, + SchemaTestCase { + description: "inferred columns first without enforcement, folded names", + columns: SchemaColumns { + yml: pairs(&[("ID", "INTEGER")]), + inferred: Some(vec![("Label".to_owned(), Some("TEXT".to_owned()))]), + warehouse: pairs(&[("id", "TEXT"), ("label", "TEXT"), ("extra_x", "INTEGER")]), + }, + comparison: SchemaComparison { + type_enforcement: false, + inferred_schema_complete: true, + case_folding: true, + }, + expected_findings: vec![ + finding( + ( + SchemaChangeKind::ColumnTypeChanged, + "id", + SchemaColumnSource::Yml, + ), + Some("INTEGER"), + Some("TEXT"), + ), + finding( + ( + SchemaChangeKind::ColumnTypeChanged, + "extra_x", + SchemaColumnSource::Yml, + ), + Some("TEXT"), + Some("INTEGER"), + ), + ], + }, + ]; + for test_case in test_cases { + let mut equal = types_equal; + let mut family = |name: &str| -> Result, ()> { + Ok(name.starts_with("extra_").then(|| "TEXT".to_owned())) + }; + let findings = detect_schema_changes( + test_case.columns, + test_case.comparison, + (&mut equal, &mut family), + ); + assert_eq!( + findings, + Ok(test_case.expected_findings), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_renamed_references_when_matching_then_returns_origin_mapping() { + let test_cases = [ + OriginTestCase { + description: "one renamed reference", + query_sql: r#"SELECT * FROM __ref("orders_v2") JOIN __ref("customers")"#, + recorded_query: r#"SELECT * FROM __ref("orders") JOIN __ref("customers")"#, + renames: vec![("orders_v2", "orders", true)], + span_names: vec!["orders_v2"], + expected_mapping: Some(pairs(&[("orders_v2", "orders")])), + }, + OriginTestCase { + description: "chained renames reach the oldest name", + query_sql: r#"SELECT * FROM __ref("orders_v3")"#, + recorded_query: r#"SELECT * FROM __ref("orders")"#, + renames: vec![ + ("orders_v3", "orders_v2", true), + ("orders_v2", "orders", true), + ], + span_names: vec!["orders_v3"], + expected_mapping: Some(pairs(&[("orders_v3", "orders")])), + }, + OriginTestCase { + description: "rename recorded after the build does not apply", + query_sql: r#"SELECT * FROM __ref("orders_v2")"#, + recorded_query: r#"SELECT * FROM __ref("orders")"#, + renames: vec![("orders_v2", "orders", false)], + span_names: vec!["orders_v2"], + expected_mapping: None, + }, + OriginTestCase { + description: "other edits are not a rename", + query_sql: r#"SELECT id FROM __ref("orders_v2")"#, + recorded_query: r#"SELECT * FROM __ref("orders")"#, + renames: vec![("orders_v2", "orders", true)], + span_names: vec!["orders_v2"], + expected_mapping: None, + }, + ]; + for test_case in test_cases { + let renames: Vec = test_case + .renames + .iter() + .map(|(new_name, origin_name, applies)| RenameCandidate { + new_name: (*new_name).to_owned(), + origin_name: (*origin_name).to_owned(), + applies: *applies, + }) + .collect(); + let found: Vec = test_case + .span_names + .iter() + .map(|name| quoted_span(test_case.query_sql, name)) + .collect(); + let mut spans = |_: &std::collections::BTreeSet| -> Result, ()> { + Ok(found.clone()) + }; + let mut hash = |text: &str| -> Result { Ok(text.to_owned()) }; + assert_eq!( + origin_reference_names( + (test_case.query_sql, test_case.recorded_query), + &renames, + (&mut spans, &mut hash), + ), + Ok(test_case.expected_mapping), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_changes/tests/test_types.rs new file mode 100644 index 0000000000..0074d77cc8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/tests/test_types.rs @@ -0,0 +1,50 @@ +use crate::planner_changes::models::{ + Backfill, BackfillFacts, ChangeKind, SchemaColumns, SchemaComparison, SchemaFindingRow, +}; + +pub(super) struct ReplayTestCase { + pub(super) description: &'static str, + pub(super) raw: Option<&'static str>, + pub(super) expected_backfill: Result, +} + +pub(super) struct MetadataTextTestCase { + pub(super) description: &'static str, + pub(super) metadata_json: &'static str, + pub(super) expected_with_hash: &'static str, + pub(super) expected_without_hash: &'static str, + pub(super) expected_recorded_hash: Option<&'static str>, +} + +pub(super) struct MetadataChangesTestCase { + pub(super) description: &'static str, + pub(super) current: &'static str, + pub(super) recorded: Option<&'static str>, + pub(super) origin_names: Option>, + pub(super) expected_functions: Vec, + pub(super) expected_config_changed: bool, +} + +pub(super) struct DecisionTestCase { + pub(super) description: &'static str, + pub(super) facts: BackfillFacts, + pub(super) replay_on_change: Option<&'static str>, + pub(super) expected_kind: ChangeKind, + pub(super) expected_backfill: Backfill, +} + +pub(super) struct SchemaTestCase { + pub(super) description: &'static str, + pub(super) columns: SchemaColumns, + pub(super) comparison: SchemaComparison, + pub(super) expected_findings: Vec, +} + +pub(super) struct OriginTestCase { + pub(super) description: &'static str, + pub(super) query_sql: &'static str, + pub(super) recorded_query: &'static str, + pub(super) renames: Vec<(&'static str, &'static str, bool)>, + pub(super) span_names: Vec<&'static str>, + pub(super) expected_mapping: Option>, +} diff --git a/crates/sqlbuild-analysis/src/planner_changes/types.rs b/crates/sqlbuild-analysis/src/planner_changes/types.rs new file mode 100644 index 0000000000..64b3278afe --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_changes/types.rs @@ -0,0 +1,23 @@ +//! Planner change-detection callback types. + +use std::collections::BTreeSet; + +use crate::planner_changes::models::NameSpan; + +/// Finds the `__ref` spans of some names. +pub type SpanFinder<'a, E> = dyn FnMut(&BTreeSet) -> Result, E> + 'a; + +/// Whether two type strings are equal (warehouse or family type first, expected type second). +pub type TypesEqual<'a, E> = dyn FnMut(&str, &str) -> Result + 'a; + +/// The type of the one dynamic family accepting a column name, if exactly one does. +pub type SingleFamilyType<'a, E> = dyn FnMut(&str) -> Result, E> + 'a; + +/// Hashes a candidate query. +pub type QueryHasher<'a, E> = dyn FnMut(&str) -> Result + 'a; + +/// The column checks schema comparison asks the caller: type equality and single-family type. +pub type SchemaChecks<'a, E> = (&'a mut TypesEqual<'a, E>, &'a mut SingleFamilyType<'a, E>); + +/// How rename matching finds `__ref` spans and hashes a candidate query. +pub type RenameMatching<'a, E> = (&'a mut SpanFinder<'a, E>, &'a mut QueryHasher<'a, E>); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index e5a1a65ec8..d8ddc827ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -44,6 +44,7 @@ pub(crate) mod macro_bridge; pub(crate) mod manifest; pub(crate) mod model_config; pub(crate) mod plan_output; +pub(crate) mod planner_changes; pub(crate) mod presentation_text; pub(crate) mod project_assembly; pub(crate) mod python_values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs new file mode 100644 index 0000000000..404fbe3599 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs @@ -0,0 +1,61 @@ +//! Per-model actions from each model's own change, and the incremental replacement predicate. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; + +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const INCREMENTAL_MATERIALIZATION: &str = "incremental"; +const CREATE_TABLE_ACTION: &str = "create_table"; + +/// Each selected model's change with the backfill its own `replay_on_change` decided. +#[pyfunction] +#[pyo3(signature = (*, scope, changes))] +fn resolve_model_actions<'py>( + scope: &Bound<'py, PyAny>, + changes: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = scope.py(); + let model_kind: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + let action_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "ResolvedModelAction")?; + let selected: Bound<'py, PyAny> = scope.getattr("selected_keys")?; + let models_by_name: Bound<'py, PyAny> = scope.getattr("models_by_name")?; + let model_changes: Bound<'py, PyAny> = changes.getattr("models")?; + let resolved: Bound<'py, PyDict> = PyDict::new(py); + for key in scope.getattr("execution_order")?.try_iter()? { + let key: Bound<'py, PyAny> = key?; + if !selected.contains(&key)? || !key.getattr("resource_type")?.eq(&model_kind)? { + continue; + } + let name: Bound<'py, PyAny> = key.getattr("name")?; + let change: Bound<'py, PyAny> = model_changes.call_method1("get", (&name,))?; + if !models_by_name.contains(&name)? || change.is_none() { + continue; + } + let backfill: Bound<'py, PyAny> = change.getattr("backfill")?; + resolved.set_item( + name, + call_with(&action_class, &[("change", change), ("backfill", backfill)])?, + )?; + } + call_with( + &imported(py, MODELS_MODULE, "PlannerResolvedActions")?, + &[("models", resolved.into_any())], + ) +} + +/// Incremental replacements, whose bounds must ignore the destination maximum. +#[pyfunction] +#[pyo3(signature = (*, materialization_type, action))] +fn replaces_incremental_relation(materialization_type: &str, action: &str) -> bool { + materialization_type == INCREMENTAL_MATERIALIZATION && action == CREATE_TABLE_ACTION +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_model_actions, module)?)?; + module.add_function(wrap_pyfunction!(replaces_incremental_relation, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs new file mode 100644 index 0000000000..372cfe5f50 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -0,0 +1,820 @@ +//! Detect how each selected model and function changed since its recorded build. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_changes::constants::{ + CASE_FOLDING_DIALECT, EMPTY_FINGERPRINT_METADATA_JSON, +}; +use sqlbuild_analysis::planner_changes::main::detect_schema_changes::detect_schema_changes; +use sqlbuild_analysis::planner_changes::main::metadata_changes::metadata_changes; +use sqlbuild_analysis::planner_changes::main::model_backfill::model_backfill; +use sqlbuild_analysis::planner_changes::main::model_change_kind::model_change_kind; +use sqlbuild_analysis::planner_changes::main::recorded_declared_columns_hash::recorded_declared_columns_hash; +use sqlbuild_analysis::planner_changes::main::replay_policy_needed::replay_policy_needed; +use sqlbuild_analysis::planner_changes::main::resolve_replay_on_change::resolve_replay_on_change; +use sqlbuild_analysis::planner_changes::main::with_declared_columns_hash::with_declared_columns_hash; +use sqlbuild_analysis::planner_changes::models::{ + Backfill, BackfillAction, BackfillFacts, MetadataChanges, SchemaColumns, SchemaComparison, + SchemaFindingRow, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_changes::renames::origin_names; +use crate::bindings::_helpers::planner_changes::results::{ + MODELS_MODULE, backfill_result, change_kind, planner_enum, replay_policy_error, schema_findings, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const MODEL_METADATA_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.model_metadata"; +const VERSION_METADATA_MODULE: &str = + "sqlbuild.compiler.planner.main.identity.version_identity_model_metadata"; +const FUNCTION_HASHES_MODULE: &str = + "sqlbuild.compiler.planner.main.identity.version_identity_function_hashes"; +const HASHING_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.hashing"; +const FUNCTIONS_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.functions"; +const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; +const TYPES_EQUAL_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; +const DYNAMIC_FAMILIES_MODULE: &str = "sqlbuild.spec.contracts.main.matching_dynamic_families"; +const DEBUG_EVENT_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; +const LOG_SQL_MODULE: &str = "sqlbuild.diagnostics.main.log_sql"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const CHANGES_LOGGER: &str = "sqlbuild.planner.changes"; +const REPLAY_CONFIG_KEY: &str = "replay_on_change"; + +/// Inputs shared by every model of one detection pass. +struct ModelPass<'py> { + snapshot: Bound<'py, PyAny>, + query_change_tracking: bool, + full_refresh: bool, + function_local_hashes: Bound<'py, PyAny>, + hook_version_hashes: Bound<'py, PyAny>, +} + +/// Selected model and function changes. +#[pyfunction] +#[pyo3(signature = (**options))] +fn detect_changes<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "detect_changes", + options, + ( + &["project", "scope", "snapshot", "full_refresh"], + &["expected_version_hashes", "expected_metadata_jsons"], + ), + )?; + let (project, scope, snapshot) = ( + arguments.value("project")?, + arguments.value("scope")?, + arguments.value("snapshot")?, + ); + let (project, scope, snapshot) = (&project, &scope, &snapshot); + let full_refresh: bool = arguments.flag("full_refresh")?; + let models: Bound<'py, PyDict> = scope_model_changes( + (project, scope, snapshot), + None, + ( + arguments.value("expected_version_hashes")?, + arguments.value("expected_metadata_jsons")?, + ), + )?; + let functions: Bound<'py, PyDict> = PyDict::new(py); + let result_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "FunctionChangeResult")?; + let fingerprint_sql_of: Bound<'py, PyAny> = imported( + py, + FUNCTIONS_MODULE, + "build_compiled_function_fingerprint_sql", + )?; + let detect_function: Bound<'py, PyAny> = + imported(py, FUNCTIONS_MODULE, "detect_function_change")?; + let selected: Bound<'py, PyAny> = scope.getattr("selected_keys")?; + for function in project.getattr("functions")?.try_iter()? { + let function: Bound<'py, PyAny> = function?; + let name: Bound<'py, PyAny> = function.getattr("name")?; + let fingerprint_sql: Bound<'py, PyAny> = fingerprint_sql_of.call1((&function,))?; + let mut arguments: Vec<(&str, Bound<'py, PyAny>)> = + vec![("fingerprint_sql", fingerprint_sql.clone())]; + if selected.contains(function.getattr("key")?)? { + let reason: Bound<'py, PyAny> = call_with( + &detect_function, + &[ + ("function", function.clone()), + ("fingerprint_sql", fingerprint_sql), + ( + "fingerprint", + snapshot + .getattr("fingerprints")? + .getattr("functions")? + .call_method1("get", (&name,))?, + ), + ( + "query_change_tracking", + project + .getattr("settings")? + .getattr("query_change_tracking")?, + ), + ( + "full_refresh", + full_refresh.into_pyobject(py)?.to_owned().into_any(), + ), + ("dialect", snapshot.getattr("column_dialect")?), + ], + )?; + arguments.push(("reason", reason)); + } + functions.set_item(name, call_with(&result_class, &arguments)?)?; + } + call_with( + &imported(py, MODELS_MODULE, "PlannerChangeResults")?, + &[ + ("models", models.into_any()), + ("functions", functions.into_any()), + ], + ) +} + +/// Selected model changes, without inspecting functions. +#[pyfunction] +#[pyo3(signature = (**options))] +fn detect_model_changes_in_scope<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "detect_model_changes_in_scope", + options, + ( + &["project", "scope", "snapshot"], + &[ + "query_change_tracking", + "expected_version_hashes", + "expected_metadata_jsons", + ], + ), + )?; + scope_model_changes( + ( + &arguments.value("project")?, + &arguments.value("scope")?, + &arguments.value("snapshot")?, + ), + arguments.value("query_change_tracking")?.extract()?, + ( + arguments.value("expected_version_hashes")?, + arguments.value("expected_metadata_jsons")?, + ), + ) +} + +/// One model's change and backfill. +#[pyfunction] +#[pyo3(signature = (**options))] +fn detect_model_changes<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "detect_model_changes", + options, + ( + &[ + "model", + "snapshot", + "sql_analysis_enabled", + "query_change_tracking", + "full_refresh", + ], + &[ + "function_local_hashes", + "hook_version_hashes", + "expected_version_hash", + "expected_metadata_json", + ], + ), + )?; + let pass = ModelPass { + snapshot: arguments.value("snapshot")?, + query_change_tracking: arguments.flag("query_change_tracking")?, + full_refresh: arguments.flag("full_refresh")?, + function_local_hashes: arguments.value("function_local_hashes")?, + hook_version_hashes: arguments.value("hook_version_hashes")?, + }; + model_change( + &pass, + &arguments.value("model")?, + ( + arguments.value("expected_version_hash")?.extract()?, + arguments.value("expected_metadata_json")?.extract()?, + ), + ) +} + +fn scope_model_changes<'py>( + (project, scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + query_change_tracking: Option, + (version_hashes, metadata_jsons): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = project.py(); + let tracking: bool = match query_change_tracking { + Some(value) => value, + None => project + .getattr("settings")? + .getattr("query_change_tracking")? + .extract()?, + }; + let function_local_hashes: Bound<'py, PyAny> = call_with( + &imported(py, FUNCTION_HASHES_MODULE, "build_function_local_hashes")?, + &[ + ("functions", project.getattr("functions")?), + ("dialect", project.getattr("sql_analysis_dialect")?), + ], + )?; + let hook_version_hashes: Bound<'py, PyDict> = PyDict::new(py); + let identities: Bound<'py, PyAny> = + imported(py, HOOK_IDENTITIES_MODULE, "build_hook_identities")? + .call1((project.getattr("hook_functions")?,))?; + for item in identities.call_method0("items")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + hook_version_hashes.set_item( + item.get_item(0)?, + item.get_item(1)?.getattr("version_hash")?, + )?; + } + let pass = ModelPass { + snapshot: snapshot.clone(), + query_change_tracking: tracking, + full_refresh: false, + function_local_hashes, + hook_version_hashes: hook_version_hashes.into_any(), + }; + let model_kind: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + let selected: Bound<'py, PyAny> = scope.getattr("selected_keys")?; + let models_by_name: Bound<'py, PyAny> = scope.getattr("models_by_name")?; + let changes: Bound<'py, PyDict> = PyDict::new(py); + for key in scope.getattr("execution_order")?.try_iter()? { + let key: Bound<'py, PyAny> = key?; + if !selected.contains(&key)? || !key.getattr("resource_type")?.eq(&model_kind)? { + continue; + } + let model: Bound<'py, PyAny> = + models_by_name.call_method1("get", (key.getattr("name")?,))?; + if model.is_none() { + continue; + } + let name: Bound<'py, PyAny> = model.getattr("name")?; + let expected: (Option, Option) = ( + lookup(&version_hashes, &name)?, + lookup(&metadata_jsons, &name)?, + ); + changes.set_item(&name, model_change(&pass, &model, expected)?)?; + } + Ok(changes) +} + +fn lookup(mapping: &Bound<'_, PyAny>, name: &Bound<'_, PyAny>) -> PyResult> { + if mapping.is_none() { + return Ok(None); + } + mapping.call_method1("get", (name,))?.extract() +} + +fn model_change<'py>( + pass: &ModelPass<'py>, + model: &Bound<'py, PyAny>, + (expected_version_hash, expected_metadata_json): (Option, Option), +) -> PyResult> { + let py: Python<'py> = model.py(); + let snapshot: &Bound<'py, PyAny> = &pass.snapshot; + let model_name: Bound<'py, PyAny> = model.getattr("name")?; + let columns_hash: String = call_with( + &imported(py, MODEL_METADATA_MODULE, "declared_columns_hash")?, + &[("model", model.clone())], + )? + .extract()?; + let base_metadata: String = match expected_metadata_json.filter(|text| !text.is_empty()) { + Some(text) => text, + None => call_with( + &imported( + py, + VERSION_METADATA_MODULE, + "build_model_version_identity_metadata_json", + )?, + &[ + ("model", model.clone()), + ("function_local_hashes", pass.function_local_hashes.clone()), + ("hook_version_hashes", pass.hook_version_hashes.clone()), + ], + )? + .extract()?, + }; + let metadata_json: String = with_declared_columns_hash(&base_metadata, &columns_hash); + let mut result = ChangeResult::new( + py, + &model_name, + &metadata_json, + expected_version_hash.as_deref(), + )?; + if pass.full_refresh { + return result.finish( + Backfill { + action: BackfillAction::Full, + duration: None, + }, + "no_change", + ); + } + let relation_exists: bool = snapshot + .getattr("existing_relations")? + .contains(&model_name)?; + let fingerprint: Bound<'py, PyAny> = snapshot + .getattr("fingerprints")? + .getattr("models")? + .call_method1("get", (&model_name,))?; + let recorded: Option = recorded_fingerprint(&fingerprint)?; + let dialect: Bound<'py, PyAny> = snapshot.getattr("column_dialect")?; + let compiled_query_hash: Option = if pass.query_change_tracking && recorded.is_some() { + Some( + call_with( + &imported(py, HASHING_MODULE, "model_definition_hash")?, + &[ + ("model_name", model_name.clone()), + ("query_sql", model.getattr("query_sql")?), + ("dialect", dialect.clone()), + ], + )? + .extract()?, + ) + } else { + None + }; + if let Some(recorded) = &recorded { + result.previous(recorded); + } + let definition_changed: bool = match (&compiled_query_hash, &recorded) { + (Some(hash), Some(recorded)) => *hash != recorded.definition_hash, + _ => false, + }; + if !relation_exists && snapshot.getattr("renamed_models")?.contains(&model_name)? { + result.query_changed = definition_changed; + let backfill: Backfill = if definition_changed { + replay_policy(model)? + } else { + Backfill { + action: BackfillAction::ForwardOnly, + duration: None, + } + }; + return result.finish(backfill, "renamed"); + } + if !relation_exists { + result.clear_previous(); + result.recorded_build_relation_missing = recorded.is_some(); + return result.finish( + Backfill { + action: BackfillAction::Full, + duration: None, + }, + "first_run", + ); + } + let recorded_metadata: Option<&str> = recorded + .as_ref() + .map(|recorded| recorded.metadata_json.as_str()) + .filter(|text| *text != EMPTY_FINGERPRINT_METADATA_JSON); + let renames: Bound<'py, PyAny> = snapshot.getattr("reference_renames")?; + let origins: Option> = match (&compiled_query_hash, &recorded) { + (Some(_), Some(recorded)) if definition_changed && renames.is_truthy()? => origin_names( + (model.getattr("query_sql")?.extract()?, &fingerprint), + &renames, + (&recorded.definition_hash, &dialect), + )?, + _ => None, + }; + let changes: MetadataChanges = metadata_changes( + (&metadata_json, recorded_metadata), + origins.as_deref(), + pass.query_change_tracking, + ); + let mut query_changed: bool = false; + if let (Some(hash), Some(recorded)) = (&compiled_query_hash, &recorded) { + query_changed = origins.is_none() && definition_changed; + log_query_check( + (model, &model_name, &fingerprint), + hash, + (&recorded.definition_hash, query_changed), + )?; + } + let findings: Vec = + model_schema_findings(model, snapshot, &model_name, &dialect)?; + let declared_columns_changed: bool = + recorded_declared_columns_hash(recorded_metadata).is_some_and(|hash| hash != columns_hash); + let facts = BackfillFacts { + query_changed, + functions_changed: !changes.changed_functions.is_empty(), + config_changed: changes.config_changed, + declared_columns_changed, + schema_changed: !findings.is_empty(), + }; + let replay: Option = if replay_policy_needed(facts) { + Some(replay_policy(model)?) + } else { + None + }; + let backfill: Backfill = model_backfill(facts, replay); + result.query_changed = query_changed; + result.config_changed = changes.config_changed; + result.findings = findings; + result.changed_functions = changes.changed_functions; + result.finish_kind(backfill, change_kind(py, model_change_kind(facts))?) +} + +/// The model's resolved `replay_on_change` policy. +fn replay_policy(model: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = model.py(); + let raw: Option = call_with( + &imported(py, NATIVE_MODULE, "get_config_str")?, + &[ + ("values", model.getattr("config")?.getattr("values")?), + ("key", PyString::new(py, REPLAY_CONFIG_KEY).into_any()), + ], + )? + .extract()?; + resolve_replay_on_change(raw.as_deref()).map_err(|error| replay_policy_error(py, error)) +} + +/// The recorded fingerprint fields change detection reads. +struct Recorded { + definition_hash: String, + metadata_json: String, + version_hash: String, +} + +fn recorded_fingerprint(fingerprint: &Bound<'_, PyAny>) -> PyResult> { + if fingerprint.is_none() { + return Ok(None); + } + Ok(Some(Recorded { + definition_hash: fingerprint.getattr("definition_hash")?.extract()?, + metadata_json: fingerprint.getattr("metadata_json")?.extract()?, + version_hash: fingerprint.getattr("version_hash")?.extract()?, + })) +} + +fn log_query_check( + (model, model_name, fingerprint): (&Bound<'_, PyAny>, &Bound<'_, PyAny>, &Bound<'_, PyAny>), + compiled_query_hash: &str, + (definition_hash, query_changed): (&str, bool), +) -> PyResult<()> { + let py: Python<'_> = model.py(); + let logger: Bound<'_, PyAny> = + imported(py, "logging", "getLogger")?.call1((CHANGES_LOGGER,))?; + let text = |value: &str| PyString::new(py, value).into_any(); + call_with( + &imported(py, DEBUG_EVENT_MODULE, "log_debug_event")?, + &[ + ("logger", logger.clone()), + ( + "message", + text(&format!( + "fingerprint comparison compiled_query_hash={compiled_query_hash} \ + fingerprint_definition_hash={definition_hash} query_changed={}", + if query_changed { "True" } else { "False" } + )), + ), + ("sqlbuild_subject", text("model")), + ("sqlbuild_name", model_name.clone()), + ("sqlbuild_event", text("query_change_check")), + ("sqlbuild_phase", text("planner")), + ( + "sqlbuild_status", + text(if query_changed { + "changed" + } else { + "unchanged" + }), + ), + ], + )?; + let log_sql: Bound<'_, PyAny> = imported(py, LOG_SQL_MODULE, "log_sql")?; + call_with( + &log_sql, + &[ + ("logger", logger.clone()), + ("sql", model.getattr("query_sql")?), + ("action", text("compiled_query")), + ], + )?; + call_with( + &log_sql, + &[ + ("logger", logger), + ("sql", fingerprint.getattr("definition")?), + ("action", text("fingerprint_definition")), + ], + )?; + Ok(()) +} + +fn model_schema_findings<'py>( + model: &Bound<'py, PyAny>, + snapshot: &Bound<'py, PyAny>, + model_name: &Bound<'py, PyAny>, + dialect: &Bound<'py, PyAny>, +) -> PyResult> { + let warehouse: Bound<'py, PyAny> = snapshot + .getattr("existing_columns")? + .call_method1("get", (model_name,))?; + if warehouse.is_none() { + return Ok(Vec::new()); + } + let schema_entry: Bound<'py, PyAny> = model.getattr("schema_entry")?; + let mut yml: Vec<(String, String)> = Vec::new(); + let mut type_enforcement: bool = false; + let mut dynamic: Option> = None; + if !schema_entry.is_none() { + for column in schema_entry.getattr("columns")?.try_iter()? { + let column: Bound<'py, PyAny> = column?; + if let Some(kind) = column.getattr("type")?.extract::>()? { + yml.push((column.getattr("name")?.extract()?, kind)); + } + } + type_enforcement = schema_entry + .getattr("type_enforcement")? + .extract::>()? + .unwrap_or(false); + let families: Bound<'py, PyAny> = schema_entry.getattr("dynamic_columns")?; + if families.is_truthy()? { + dynamic = Some(families); + } + } + let inferred: Option)>> = { + let columns: Bound<'py, PyAny> = model.getattr("inferred_columns")?; + if columns.is_none() { + None + } else { + let mut items: Vec<(String, Option)> = Vec::new(); + for column in columns.try_iter()? { + let column: Bound<'py, PyAny> = column?; + items.push(( + column.getattr("name")?.extract()?, + column.getattr("type")?.extract()?, + )); + } + Some(items) + } + }; + if yml.is_empty() && inferred.as_ref().is_none_or(Vec::is_empty) { + return Ok(Vec::new()); + } + let inferred_schema_complete: bool = !model.getattr("fast_lineage_has_star")?.is_truthy()?; + schema_findings_of( + SchemaColumns { + yml, + inferred, + warehouse: column_pairs(&warehouse)?, + }, + (type_enforcement, inferred_schema_complete), + (dynamic, dialect), + ) +} + +fn column_pairs(columns: &Bound<'_, PyAny>) -> PyResult> { + let mut pairs: Vec<(String, String)> = Vec::new(); + for column in columns.try_iter()? { + let column: Bound<'_, PyAny> = column?; + pairs.push(( + column.getattr("name")?.extract()?, + column.getattr("type")?.extract()?, + )); + } + Ok(pairs) +} + +fn schema_findings_of<'py>( + columns: SchemaColumns, + (type_enforcement, inferred_schema_complete): (bool, bool), + (dynamic, dialect): (Option>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = dialect.py(); + let dialect_name: Option = dialect.extract()?; + let types_equal_function: Bound<'py, PyAny> = imported(py, TYPES_EQUAL_MODULE, "types_equal")?; + let mut types_equal = |left: &str, right: &str| -> PyResult { + call_with( + &types_equal_function, + &[ + ("left", PyString::new(py, left).into_any()), + ("right", PyString::new(py, right).into_any()), + ("dialect", dialect.clone()), + ], + )? + .is_truthy() + }; + let matching_function: Bound<'py, PyAny> = + imported(py, DYNAMIC_FAMILIES_MODULE, "matching_dynamic_families")?; + let mut single_family_type = |name: &str| -> PyResult> { + let Some(families) = &dynamic else { + return Ok(None); + }; + let matching: Bound<'py, PyAny> = call_with( + &matching_function, + &[ + ("families", families.clone()), + ("column_name", PyString::new(py, name).into_any()), + ], + )?; + if matching.len()? != 1 { + return Ok(None); + } + matching.get_item(0)?.getattr("type")?.extract() + }; + detect_schema_changes( + columns, + SchemaComparison { + type_enforcement, + inferred_schema_complete, + case_folding: dialect_name.as_deref() == Some(CASE_FOLDING_DIALECT), + }, + (&mut types_equal, &mut single_family_type), + ) +} + +/// Schema findings between declared or inferred columns and warehouse columns. +#[pyfunction(name = "detect_schema_changes")] +#[pyo3(signature = (**options))] +fn schema_changes<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "detect_schema_changes", + options, + ( + &[ + "yml_columns", + "inferred_columns", + "warehouse_columns", + "type_enforcement", + "inferred_schema_complete", + ], + &["dynamic_columns", "dialect"], + ), + )?; + let yml_columns: Bound<'py, PyAny> = arguments.value("yml_columns")?; + let inferred_columns: Bound<'py, PyAny> = arguments.value("inferred_columns")?; + let warehouse_columns: Bound<'py, PyAny> = arguments.value("warehouse_columns")?; + let type_enforcement: bool = arguments.flag("type_enforcement")?; + let inferred_schema_complete: bool = arguments.flag("inferred_schema_complete")?; + let dynamic_columns: Bound<'py, PyAny> = arguments.value("dynamic_columns")?; + let dialect: Bound<'py, PyAny> = arguments.value("dialect")?; + let py: Python<'py> = yml_columns.py(); + let inferred: Option)>> = if inferred_columns.is_none() { + None + } else { + let mut items: Vec<(String, Option)> = Vec::new(); + for column in inferred_columns.try_iter()? { + let column: Bound<'py, PyAny> = column?; + items.push(( + column.getattr("name")?.extract()?, + column.getattr("type")?.extract()?, + )); + } + Some(items) + }; + let dynamic: Option> = if dynamic_columns.is_truthy()? { + Some(dynamic_columns) + } else { + None + }; + let findings: Vec = schema_findings_of( + SchemaColumns { + yml: column_pairs(&yml_columns)?, + inferred, + warehouse: column_pairs(&warehouse_columns)?, + }, + (type_enforcement, inferred_schema_complete), + (dynamic, &dialect), + )?; + schema_findings(py, &findings) +} + +/// The `ChangeDetectionResult` being assembled for one model. +struct ChangeResult<'py> { + py: Python<'py>, + model_name: Bound<'py, PyAny>, + metadata_json: String, + version_hash: Option, + previous_metadata_json: Option, + previous_version_hash: Option, + query_changed: bool, + config_changed: bool, + recorded_build_relation_missing: bool, + findings: Vec, + changed_functions: Vec, +} + +impl<'py> ChangeResult<'py> { + fn new( + py: Python<'py>, + model_name: &Bound<'py, PyAny>, + metadata_json: &str, + version_hash: Option<&str>, + ) -> PyResult { + Ok(Self { + py, + model_name: model_name.clone(), + metadata_json: metadata_json.to_owned(), + version_hash: version_hash.map(str::to_owned), + previous_metadata_json: None, + previous_version_hash: None, + query_changed: false, + config_changed: false, + recorded_build_relation_missing: false, + findings: Vec::new(), + changed_functions: Vec::new(), + }) + } + + fn previous(&mut self, recorded: &Recorded) { + self.previous_metadata_json = Some(recorded.metadata_json.clone()); + self.previous_version_hash = Some(recorded.version_hash.clone()); + } + + fn clear_previous(&mut self) { + self.previous_metadata_json = None; + self.previous_version_hash = None; + } + + fn finish(self, backfill: Backfill, kind: &str) -> PyResult> { + let kind: Bound<'py, PyAny> = planner_enum(self.py, "ChangeKind", kind)?; + self.finish_kind(backfill, kind) + } + + fn finish_kind( + self, + backfill: Backfill, + kind: Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = self.py; + let flag = |value: bool| { + value + .into_pyobject(py) + .map(|flag| flag.to_owned().into_any()) + }; + call_with( + &imported(py, MODELS_MODULE, "ChangeDetectionResult")?, + &[ + ("model_name", self.model_name), + ("change_kind", kind), + ("query_changed", flag(self.query_changed)?), + ("config_changed", flag(self.config_changed)?), + ( + "fingerprint_metadata_json", + self.metadata_json.into_pyobject(py)?.into_any(), + ), + ( + "previous_metadata_json", + self.previous_metadata_json.into_pyobject(py)?.into_any(), + ), + ( + "fingerprint_version_hash", + self.version_hash.into_pyobject(py)?.into_any(), + ), + ( + "previous_version_hash", + self.previous_version_hash.into_pyobject(py)?.into_any(), + ), + ( + "schema_findings", + schema_findings(py, &self.findings)?.into_any(), + ), + ("backfill", backfill_result(py, &backfill)?), + ( + "recorded_build_relation_missing", + flag(self.recorded_build_relation_missing)?, + ), + ( + "changed_functions", + pyo3::types::PyTuple::new(py, self.changed_functions)?.into_any(), + ), + ], + ) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(detect_changes, module)?)?; + module.add_function(wrap_pyfunction!(detect_model_changes_in_scope, module)?)?; + module.add_function(wrap_pyfunction!(detect_model_changes, module)?)?; + module.add_function(wrap_pyfunction!(schema_changes, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs new file mode 100644 index 0000000000..e760bb22da --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs @@ -0,0 +1,46 @@ +//! Version-identity metadata helpers the remaining Python planner reads. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_changes::main::version_identity_payload::version_identity_payload; +use sqlbuild_analysis::planner_changes::main::with_declared_columns_hash::with_declared_columns_hash as with_hash; +use sqlbuild_analysis::planner_changes::main::without_declared_columns_hash::without_declared_columns_hash as without_hash; +use sqlbuild_analysis::planner_changes::models::IdentityPayload; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +/// The decoded metadata that participates in version identity, the raw text when it is not JSON, or `None`. +#[pyfunction] +#[pyo3(signature = (metadata_json))] +fn version_identity_metadata_payload<'py>( + py: Python<'py>, + metadata_json: Option<&str>, +) -> PyResult> { + match version_identity_payload(metadata_json, false) { + IdentityPayload::Absent => Ok(py.None().into_bound(py)), + IdentityPayload::Raw(text) => Ok(text.into_pyobject(py)?.into_any()), + IdentityPayload::Object(object) => { + imported(py, "json", "loads")?.call1((serde_json::Value::Object(object).to_string(),)) + } + } +} + +/// Fingerprint metadata JSON that also records the model's declared-columns hash. +#[pyfunction] +#[pyo3(signature = (*, metadata_json, declared_columns_hash))] +fn with_declared_columns_hash(metadata_json: &str, declared_columns_hash: &str) -> String { + with_hash(metadata_json, declared_columns_hash) +} + +/// Fingerprint metadata JSON without the declared-columns hash key, if present. +#[pyfunction] +fn without_declared_columns_hash(metadata_json: &str) -> String { + without_hash(metadata_json) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(version_identity_metadata_payload, module)?)?; + module.add_function(wrap_pyfunction!(with_declared_columns_hash, module)?)?; + module.add_function(wrap_pyfunction!(without_declared_columns_hash, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/mod.rs new file mode 100644 index 0000000000..301a6f231c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod actions; +pub(crate) mod detect; +pub(crate) mod metadata; +pub(crate) mod registration; +pub(crate) mod renames; +pub(crate) mod results; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/registration.rs new file mode 100644 index 0000000000..750b787720 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/registration.rs @@ -0,0 +1,12 @@ +//! Register planner change detection. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::planner_changes::{actions, detect, metadata, renames}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + actions::register(module)?; + detect::register(module)?; + metadata::register(module)?; + renames::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs new file mode 100644 index 0000000000..d19d59345b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs @@ -0,0 +1,275 @@ +//! Model renames that downstream `__ref` names may have been rewritten for. + +use std::collections::BTreeSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_changes::main::origin_reference_names::origin_reference_names; +use sqlbuild_analysis::planner_changes::models::{NameSpan, RenameCandidate}; +use sqlbuild_rules::lint_text::main::neutralize_interpolation::neutralize_interpolation; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const HASHING_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.hashing"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; +const GENERIC_DIALECT: &str = "generic"; +const FINAL_NEWLINE: &str = "\n"; + +const REF_CALL_PREFIX: &str = "__ref"; + +/// The rename mapping under which the model query hashes to its recorded definition. +pub(crate) fn origin_names( + (query_sql, fingerprint): (String, &Bound<'_, PyAny>), + renames: &Bound<'_, PyAny>, + (recorded_hash, dialect): (&str, &Bound<'_, PyAny>), +) -> PyResult>> { + let py: Python<'_> = fingerprint.py(); + let recorded_at: Bound<'_, PyAny> = utc(&fingerprint.getattr("ts")?)?; + let mut candidates: Vec = Vec::new(); + for rename in renames.try_iter()? { + let rename: Bound<'_, PyAny> = rename?; + let renamed_at: Bound<'_, PyAny> = rename.getattr("recorded_at")?; + candidates.push(RenameCandidate { + new_name: rename.getattr("new_name")?.extract()?, + origin_name: rename.getattr("origin_name")?.extract()?, + applies: renamed_at.is_none() || recorded_at.lt(utc(&renamed_at)?)?, + }); + } + let dialect_name: String = dialect + .extract::>()? + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| GENERIC_DIALECT.to_owned()); + let mut spans = |names: &BTreeSet| ref_name_spans(&query_sql, &dialect_name, names); + let compute_hash: Bound<'_, PyAny> = imported(py, NATIVE_MODULE, "compute_query_hash")?; + let mut query_hash = |text: &str| -> PyResult { + call_with( + &compute_hash, + &[ + ("query_sql", PyString::new(py, text).into_any()), + ("dialect", dialect.clone()), + ], + )? + .extract() + }; + origin_reference_names( + (&query_sql, recorded_hash), + &candidates, + (&mut spans, &mut query_hash), + ) +} + +fn ref_name_spans( + query_sql: &str, + dialect: &str, + names: &BTreeSet, +) -> PyResult> { + let neutralized = neutralize_interpolation(query_sql, dialect, false).map_err(value_error)?; + let mut spans: Vec = Vec::new(); + for site in neutralized.sites { + let Some((offset, name)) = ref_call_name(&site.original_text) else { + continue; + }; + if !names.contains(name) { + continue; + } + let start: usize = site.original_start + site.original_text[..offset].chars().count(); + spans.push(NameSpan { + start, + end: start + name.chars().count(), + name: name.to_owned(), + }); + } + Ok(spans) +} + +/// The byte offset and name of a whole `__ref('name')` or `__ref("name")` call. +fn ref_call_name(text: &str) -> Option<(usize, &str)> { + let rest: &str = text.strip_prefix(REF_CALL_PREFIX)?.trim_start(); + let rest: &str = rest.strip_prefix('(')?.trim_start(); + let quote: char = rest + .chars() + .next() + .filter(|quote| matches!(quote, '\'' | '"'))?; + let body: &str = &rest[1..]; + let length: usize = body.find(['\'', '"'])?; + let name: &str = &body[..length]; + let tail: &str = body[length..].strip_prefix(quote)?.trim_start(); + let tail: &str = tail.strip_prefix(')')?; + if name.is_empty() || !(tail.is_empty() || tail == FINAL_NEWLINE) { + return None; + } + Some((text.len() - body.len(), name)) +} + +/// `value` in UTC; a naive value is taken as UTC. +fn utc<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + let zone: Bound<'py, PyAny> = imported(py, "datetime", "UTC")?; + if value.getattr("tzinfo")?.is_none() || value.call_method0("utcoffset")?.is_none() { + return call_with(&value.getattr("replace")?, &[("tzinfo", zone)]); + } + value.call_method1("astimezone", (zone,)) +} + +/// The rename mapping under which `query_sql` hashes to the recorded definition, or `None`. +#[pyfunction(name = "origin_reference_names")] +#[pyo3(signature = (*, query_sql, recorded, renames, dialect))] +fn origin_reference_names_of<'py>( + query_sql: String, + recorded: &Bound<'py, PyAny>, + renames: &Bound<'py, PyAny>, + dialect: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = recorded.py(); + let recorded_hash: String = recorded.getattr("definition_hash")?.extract()?; + let Some(mapping) = origin_names((query_sql, recorded), renames, (&recorded_hash, dialect))? + else { + return Ok(None); + }; + let result: Bound<'py, PyDict> = PyDict::new(py); + for (name, origin) in mapping { + result.set_item(name, origin)?; + } + Ok(Some(result)) +} + +/// This plan's renames plus recorded renames, read only when a selected query that references models changed. +#[pyfunction] +#[pyo3(signature = (**options))] +fn plan_reference_renames<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "plan_reference_renames", + options, + ( + &[ + "runtime", "scope", "snapshot", "state", "schemas", "planned", + ], + &[], + ), + )?; + let (runtime, scope, snapshot) = ( + arguments.value("runtime")?, + arguments.value("scope")?, + arguments.value("snapshot")?, + ); + let (state, schemas, planned) = ( + arguments.value("state")?, + arguments.value("schemas")?, + arguments.value("planned")?, + ); + let (runtime, scope, snapshot) = (&runtime, &scope, &snapshot); + if !changed_referencing_query(scope, snapshot)? { + return Ok(PyTuple::empty(py)); + } + let state_schemas: Bound<'py, PyAny> = snapshot.getattr("migration_state_schemas")?; + let inspected: Bound<'py, PySet> = PySet::empty(py)?; + for schema in schemas.try_iter()? { + let schema: Bound<'py, PyAny> = schema?; + if state_schemas.contains(schema.call_method0("lower")?)? { + inspected.add(schema)?; + } + } + call_with( + &state.getattr("inspect_migration_events")?, + &[("schemas", inspected.into_any())], + )?; + let target_name: Bound<'py, PyAny> = runtime + .getattr("project")? + .getattr("effective_target_name")?; + let rename_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "ReferenceRename")?; + let mut renames: Vec> = Vec::new(); + for item in planned.call_method0("items")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + let (new_name, origin_name) = (item.get_item(0)?, item.get_item(1)?); + if new_name.ne(&origin_name)? { + renames.push(call_with( + &rename_class, + &[("new_name", new_name), ("origin_name", origin_name)], + )?); + } + } + for event in state.getattr("events")?.try_iter()? { + let event: Bound<'py, PyAny> = event?; + let event_target: Bound<'py, PyAny> = event.getattr("target_name")?; + let origin: Bound<'py, PyAny> = event.getattr("origin_model")?; + let destination: Bound<'py, PyAny> = event.getattr("destination_model")?; + if (event_target.is_none() || event_target.eq(&target_name)?) + && !origin.is_none() + && origin.ne(&destination)? + { + renames.push(call_with( + &rename_class, + &[ + ("new_name", destination), + ("origin_name", origin), + ("recorded_at", event.getattr("created_at")?), + ], + )?); + } + } + PyTuple::new(py, renames) +} + +fn changed_referencing_query( + scope: &Bound<'_, PyAny>, + snapshot: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = scope.py(); + let models_by_name: Bound<'_, PyAny> = scope.getattr("models_by_name")?; + let mut selected: Vec> = Vec::new(); + for key in scope.getattr("selected_keys")?.try_iter()? { + let model: Bound<'_, PyAny> = + models_by_name.call_method1("get", (key?.getattr("name")?,))?; + if !model.is_none() { + selected.push(model); + } + } + let recorded_models: Bound<'_, PyAny> = snapshot.getattr("fingerprints")?.getattr("models")?; + let ref_kind: Bound<'_, PyAny> = + imported(py, REFERENCE_TYPES_MODULE, "SqlReferenceKind")?.getattr("REF")?; + let definition_hash: Bound<'_, PyAny> = imported(py, HASHING_MODULE, "model_definition_hash")?; + for model in selected { + let recorded: Bound<'_, PyAny> = + recorded_models.call_method1("get", (model.getattr("name")?,))?; + if recorded.is_none() { + continue; + } + let mut references_models: bool = false; + for reference in model.getattr("references")?.try_iter()? { + if reference?.getattr("ref_kind")?.eq(&ref_kind)? { + references_models = true; + break; + } + } + if references_models + && call_with( + &definition_hash, + &[ + ("model_name", model.getattr("name")?), + ("query_sql", model.getattr("query_sql")?), + ("dialect", snapshot.getattr("column_dialect")?), + ], + )? + .ne(recorded.getattr("definition_hash")?)? + { + return Ok(true); + } + } + Ok(false) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(plan_reference_renames, module)?)?; + module.add_function(wrap_pyfunction!(origin_reference_names_of, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs new file mode 100644 index 0000000000..113cab7398 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs @@ -0,0 +1,109 @@ +//! The planner's Python change-detection result models. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use pyo3::{IntoPyObject, PyErr}; + +use sqlbuild_analysis::planner_changes::errors::UnknownReplayPolicy; +use sqlbuild_analysis::planner_changes::models::{ + Backfill, BackfillAction, ChangeKind, SchemaChangeKind, SchemaColumnSource, SchemaFindingRow, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +pub(crate) const MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +pub(crate) const TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; + +/// `PlannerInputError(message)` for an unknown replay policy. +pub(crate) fn replay_policy_error(py: Python<'_>, error: UnknownReplayPolicy) -> PyErr { + match imported(py, EXCEPTIONS_MODULE, "PlannerInputError") + .and_then(|class| class.call1((error.message(),))) + { + Ok(value) => PyErr::from_value(value), + Err(error) => error, + } +} + +/// `TYPES_MODULE.(value)`. +pub(crate) fn planner_enum<'py>( + py: Python<'py>, + name: &str, + value: &str, +) -> PyResult> { + imported(py, TYPES_MODULE, name)?.call1((value,)) +} + +/// `BackfillResult(action=..., duration=...)`. +pub(crate) fn backfill_result<'py>( + py: Python<'py>, + backfill: &Backfill, +) -> PyResult> { + let action: &str = match backfill.action { + BackfillAction::Full => "full", + BackfillAction::Bounded => "bounded", + BackfillAction::ForwardOnly => "forward", + }; + call_with( + &imported(py, MODELS_MODULE, "BackfillResult")?, + &[ + ("action", planner_enum(py, "BackfillAction", action)?), + ( + "duration", + backfill.duration.clone().into_pyobject(py)?.into_any(), + ), + ], + ) +} + +/// The `ChangeKind` member of a decided change. +pub(crate) fn change_kind<'py>(py: Python<'py>, kind: ChangeKind) -> PyResult> { + let value: &str = match kind { + ChangeKind::QueryChanged => "query_changed", + ChangeKind::FunctionChanged => "function_changed", + ChangeKind::SchemaChanged => "schema_changed", + ChangeKind::ConfigChanged => "config_changed", + ChangeKind::NoChange => "no_change", + }; + planner_enum(py, "ChangeKind", value) +} + +/// A tuple of `SchemaFinding` values. +pub(crate) fn schema_findings<'py>( + py: Python<'py>, + findings: &[SchemaFindingRow], +) -> PyResult> { + let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "SchemaFinding")?; + let mut items: Vec> = Vec::with_capacity(findings.len()); + for finding in findings { + let kind: &str = match finding.kind { + SchemaChangeKind::ColumnAdded => "column_added", + SchemaChangeKind::ColumnRemoved => "column_removed", + SchemaChangeKind::ColumnTypeChanged => "column_type_changed", + }; + let source: &str = match finding.source { + SchemaColumnSource::Yml => "yml", + SchemaColumnSource::SqlAnalysis => "sql_analysis", + }; + items.push(call_with( + &class, + &[ + ("kind", planner_enum(py, "SchemaChangeKind", kind)?), + ( + "column_name", + finding.column_name.clone().into_pyobject(py)?.into_any(), + ), + ("source", planner_enum(py, "SchemaColumnSource", source)?), + ( + "expected_type", + finding.expected_type.clone().into_pyobject(py)?.into_any(), + ), + ( + "actual_type", + finding.actual_type.clone().into_pyobject(py)?.into_any(), + ), + ], + )?); + } + PyTuple::new(py, items) +} diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 3037c000d6..167ebb5f47 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -7,9 +7,9 @@ use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, contracts, cursor, dag_artifact, diagnostics, discovery, fingerprints, frontier, graph, - lineage, macro_bridge, manifest, model_config, plan_output, project_assembly, refactor, - references, render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, - state_rows, type_system, + lineage, macro_bridge, manifest, model_config, plan_output, planner_changes, project_assembly, + refactor, references, render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, + sqltext, state_rows, type_system, }; use crate::bindings::models; @@ -45,6 +45,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { attachments::registration::register(module)?; refactor::registration::register(module)?; plan_output::registration::register(module)?; + planner_changes::registration::register(module)?; references::registration::register(module)?; authored_values::registration::register(module)?; auditing::registration::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 668642efac..789ed7c1cb 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -180,17 +180,28 @@ from sqlbuild.compiler.pipeline.models import ( PythonRunPlanOutputs, StaticCommandContext, ) +from sqlbuild.compiler.planner.classes.migration_state_inspection import ( + MigrationStateInspection, +) from sqlbuild.compiler.planner.models import ( AuditPlanEntry, + ChangeDetectionResult, CursorBounds, DeferralInputs, FutureCursorSafetyEvidence, MaximumStartSafetyEvidence, ModelPlanEntry, + PlannerChangeResults, + PlannerResolvedActions, + PlannerRuntime, + PlannerScope, PlanOutput, + ReferenceRename, + SchemaFinding, SqlTestPlanEntry, + WarehouseSnapshot, ) -from sqlbuild.compiler.planner.types import CursorGrain +from sqlbuild.compiler.planner.types import CursorGrain, MaterializationType, PlanAction from sqlbuild.compiler.project_assembly.models import NativeProjectResources from sqlbuild.compiler.python_nodes.models import ( PythonNodeGraph, @@ -3749,6 +3760,71 @@ def cursor_bound_display( *, value: str, cursor_type: str | None, cursor_grain: str | None ) -> str: ... def model_materialization_label(entry: ModelPlanEntry) -> str: ... +def detect_changes( + *, + project: CompiledProjectView, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + full_refresh: bool, + expected_version_hashes: dict[str, str] | None = None, + expected_metadata_jsons: dict[str, str] | None = None, +) -> PlannerChangeResults: ... +def detect_model_changes_in_scope( + *, + project: CompiledProjectView, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + query_change_tracking: bool | None = None, + expected_version_hashes: dict[str, str] | None = None, + expected_metadata_jsons: dict[str, str] | None = None, +) -> dict[str, ChangeDetectionResult]: ... +def detect_model_changes( + *, + model: CompiledModelView, + snapshot: WarehouseSnapshot, + sql_analysis_enabled: bool, + query_change_tracking: bool, + full_refresh: bool, + function_local_hashes: dict[str, str] | None = None, + hook_version_hashes: dict[str, str] | None = None, + expected_version_hash: str | None = None, + expected_metadata_json: str | None = None, +) -> ChangeDetectionResult: ... +def resolve_model_actions( + *, scope: PlannerScope, changes: PlannerChangeResults +) -> PlannerResolvedActions: ... +def plan_reference_renames( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + state: MigrationStateInspection, + schemas: set[str], + planned: dict[str, str], +) -> tuple[ReferenceRename, ...]: ... +def replaces_incremental_relation( + *, materialization_type: MaterializationType, action: PlanAction +) -> bool: ... +def version_identity_metadata_payload(metadata_json: str | None) -> Any: ... +def detect_schema_changes( + *, + yml_columns: tuple[ColumnInfo, ...], + inferred_columns: tuple[InferredColumn, ...] | None, + warehouse_columns: tuple[ColumnInfo, ...], + type_enforcement: bool, + inferred_schema_complete: bool, + dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] | None = None, + dialect: str | None = None, +) -> tuple[SchemaFinding, ...]: ... +def origin_reference_names( + *, + query_sql: str, + recorded: Fingerprint, + renames: tuple[ReferenceRename, ...], + dialect: str | None, +) -> dict[str, str] | None: ... +def with_declared_columns_hash(*, metadata_json: str, declared_columns_hash: str) -> str: ... +def without_declared_columns_hash(metadata_json: str) -> str: ... def model_resource_type(entry: ModelPlanEntry | None) -> str: ... def model_execution_annotation(entry: ModelPlanEntry | None) -> str: ... def materialization_type_display(resource_kind: ExecutionResourceKind) -> str: ... diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/__init__.py b/src/sqlbuild/compiler/planner/_helpers/changes/__init__.py deleted file mode 100644 index cf71b4f966..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Change detection and backfill policy resolution.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/actions.py b/src/sqlbuild/compiler/planner/_helpers/changes/actions.py deleted file mode 100644 index 2e870519c2..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/actions.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Per-model action resolution from each model's own change.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ( - ChangeDetectionResult, - PlannerChangeResults, - PlannerResolvedActions, - PlannerScope, - ResolvedModelAction, -) - - -def resolve_model_actions( - *, - scope: PlannerScope, - changes: PlannerChangeResults, -) -> PlannerResolvedActions: - """Resolve each selected model's backfill from its own change and replay_on_change only.""" - - resolved: dict[str, ResolvedModelAction] = {} - key: CompiledObjectKey - for key in scope.execution_order: - if key not in scope.selected_keys or key.resource_type != CompiledResourceType.MODEL: - continue - change: ChangeDetectionResult | None = changes.models.get(key.name) - if key.name not in scope.models_by_name or change is None: - continue - resolved[key.name] = ResolvedModelAction(change=change, backfill=change.backfill) - return PlannerResolvedActions(models=resolved) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py deleted file mode 100644 index dd611be592..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py +++ /dev/null @@ -1,375 +0,0 @@ -"""Per-model change detection orchestration.""" - -from __future__ import annotations - -import logging -from collections.abc import Callable - -from sqlbuild._native import get_config_str -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - InferredColumn, -) -from sqlbuild.compiler.compile.types import ( - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.changes.metadata import ( - changed_local_function_names, - non_function_identity_metadata_payload, - recorded_declared_columns_hash, - version_identity_metadata_payload, - with_declared_columns_hash, - with_origin_cursor_input_names, -) -from sqlbuild.compiler.planner._helpers.changes.policy import ( - pick_more_aggressive, - resolve_replay_on_change, -) -from sqlbuild.compiler.planner._helpers.changes.query import detect_query_change -from sqlbuild.compiler.planner._helpers.changes.reference_renames import origin_reference_names -from sqlbuild.compiler.planner._helpers.changes.schema import detect_schema_changes -from sqlbuild.compiler.planner._helpers.identity.functions import ( - build_compiled_function_fingerprint_sql, - detect_function_change, -) -from sqlbuild.compiler.planner._helpers.identity.hashing import model_definition_hash -from sqlbuild.compiler.planner._helpers.identity.model_metadata import declared_columns_hash -from sqlbuild.compiler.planner.constants import EMPTY_FINGERPRINT_METADATA_JSON -from sqlbuild.compiler.planner.main.identity.version_identity_function_hashes import ( - build_function_local_hashes, -) -from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( - build_model_version_identity_metadata_json, -) -from sqlbuild.compiler.planner.models import ( - BackfillResult, - ChangeDetectionResult, - FunctionChangeResult, - PlannerChangeResults, - PlannerScope, - SchemaFinding, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind -from sqlbuild.compiler.python_nodes.main._hook_identities import build_hook_identities -from sqlbuild.diagnostics.main.log_debug_event import log_debug_event -from sqlbuild.diagnostics.main.log_sql import log_sql - - -def detect_changes( - *, - project: CompiledProjectView, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - full_refresh: bool, - expected_version_hashes: dict[str, str] | None = None, - expected_metadata_jsons: dict[str, str] | None = None, -) -> PlannerChangeResults: - """Detect selected model and function changes.""" - - model_changes: dict[str, ChangeDetectionResult] = detect_model_changes_in_scope( - project=project, - scope=scope, - snapshot=snapshot, - expected_version_hashes=expected_version_hashes, - expected_metadata_jsons=expected_metadata_jsons, - ) - - function_changes: dict[str, FunctionChangeResult] = {} - function: CompiledFunctionView - for function in project.functions: - fingerprint_sql: str = build_compiled_function_fingerprint_sql(function) - if function.key not in scope.selected_keys: - function_changes[function.name] = FunctionChangeResult( - fingerprint_sql=fingerprint_sql, - ) - continue - function_changes[function.name] = FunctionChangeResult( - fingerprint_sql=fingerprint_sql, - reason=detect_function_change( - function=function, - fingerprint_sql=fingerprint_sql, - fingerprint=snapshot.fingerprints.functions.get(function.name), - query_change_tracking=project.settings.query_change_tracking, - full_refresh=full_refresh, - dialect=snapshot.column_dialect, - ), - ) - - return PlannerChangeResults(models=model_changes, functions=function_changes) - - -def detect_model_changes_in_scope( - *, - project: CompiledProjectView, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - query_change_tracking: bool | None = None, - expected_version_hashes: dict[str, str] | None = None, - expected_metadata_jsons: dict[str, str] | None = None, -) -> dict[str, ChangeDetectionResult]: - """Detect changes for selected models without inspecting functions.""" - - model_changes: dict[str, ChangeDetectionResult] = {} - effective_query_change_tracking: bool = ( - project.settings.query_change_tracking - if query_change_tracking is None - else query_change_tracking - ) - function_local_hashes: dict[str, str] = build_function_local_hashes( - functions=project.functions, dialect=project.sql_analysis_dialect - ) - hook_version_hashes: dict[str, str] = { - name: identity.version_hash - for name, identity in build_hook_identities(project.hook_functions).items() - } - key: CompiledObjectKey - for key in scope.execution_order: - if key not in scope.selected_keys or key.resource_type != CompiledResourceType.MODEL: - continue - model: CompiledModelView | None = scope.models_by_name.get(key.name) - if model is None: - continue - model_changes[model.name] = detect_model_changes( - model=model, - snapshot=snapshot, - sql_analysis_enabled=project.settings.sql_analysis, - query_change_tracking=effective_query_change_tracking, - full_refresh=False, - function_local_hashes=function_local_hashes, - hook_version_hashes=hook_version_hashes, - expected_version_hash=(expected_version_hashes or {}).get(model.name), - expected_metadata_json=(expected_metadata_jsons or {}).get(model.name), - ) - return model_changes - - -def detect_model_changes( - *, - model: CompiledModelView, - snapshot: WarehouseSnapshot, - sql_analysis_enabled: bool, - query_change_tracking: bool, - full_refresh: bool, - function_local_hashes: dict[str, str] | None = None, - hook_version_hashes: dict[str, str] | None = None, - expected_version_hash: str | None = None, - expected_metadata_json: str | None = None, -) -> ChangeDetectionResult: - """Detect changes for one model and resolve backfill policy.""" - - model_name: str = model.name - current_columns_hash: str = declared_columns_hash(model=model) - metadata_json: str = with_declared_columns_hash( - metadata_json=expected_metadata_json - or build_model_version_identity_metadata_json( - model=model, - function_local_hashes=function_local_hashes, - hook_version_hashes=hook_version_hashes, - ), - declared_columns_hash=current_columns_hash, - ) - - if full_refresh: - return ChangeDetectionResult( - model_name=model_name, - change_kind=ChangeKind.NO_CHANGE, - backfill=BackfillResult(action=BackfillAction.FULL), - fingerprint_metadata_json=metadata_json, - fingerprint_version_hash=expected_version_hash, - ) - - relation_exists: bool = model_name in snapshot.existing_relations - fingerprint: Fingerprint | None = snapshot.fingerprints.models.get(model_name) - compiled_query_hash: str | None = ( - model_definition_hash( - model_name=model_name, query_sql=model.query_sql, dialect=snapshot.column_dialect - ) - if query_change_tracking and fingerprint is not None - else None - ) - - if not relation_exists and model_name in snapshot.renamed_models: - renamed_query_changed: bool = ( - compiled_query_hash is not None - and fingerprint is not None - and detect_query_change( - compiled_query_hash=compiled_query_hash, fingerprint=fingerprint - ) - ) - return ChangeDetectionResult( - model_name=model_name, - change_kind=ChangeKind.RENAMED, - query_changed=renamed_query_changed, - backfill=( - resolve_replay_on_change( - replay_on_change=get_config_str( - values=model.config.values, key="replay_on_change" - ) - ) - if renamed_query_changed - else BackfillResult(action=BackfillAction.FORWARD_ONLY) - ), - fingerprint_metadata_json=metadata_json, - previous_metadata_json=fingerprint.metadata_json if fingerprint is not None else None, - fingerprint_version_hash=expected_version_hash, - previous_version_hash=fingerprint.version_hash if fingerprint is not None else None, - ) - if not relation_exists: - return ChangeDetectionResult( - model_name=model_name, - change_kind=ChangeKind.FIRST_RUN, - backfill=BackfillResult(action=BackfillAction.FULL), - fingerprint_metadata_json=metadata_json, - fingerprint_version_hash=expected_version_hash, - recorded_build_relation_missing=fingerprint is not None, - ) - - query_changed: bool = False - recorded_metadata_json: str | None = ( - fingerprint.metadata_json - if fingerprint is not None and fingerprint.metadata_json != EMPTY_FINGERPRINT_METADATA_JSON - else None - ) - changed_functions: tuple[str, ...] = ( - changed_local_function_names( - metadata_json=metadata_json, previous_metadata_json=recorded_metadata_json - ) - if query_change_tracking and recorded_metadata_json is not None - else () - ) - identity_payload: Callable[[str | None], object] = ( - non_function_identity_metadata_payload - if changed_functions - else version_identity_metadata_payload - ) - origin_names: dict[str, str] | None = ( - origin_reference_names( - query_sql=model.query_sql, - recorded=fingerprint, - renames=snapshot.reference_renames, - dialect=snapshot.column_dialect, - ) - if compiled_query_hash is not None - and fingerprint is not None - and snapshot.reference_renames - and compiled_query_hash != fingerprint.definition_hash - else None - ) - config_changed: bool = recorded_metadata_json is not None and identity_payload( - with_origin_cursor_input_names(metadata_json=metadata_json, renamed_refs=origin_names) - if origin_names is not None - else metadata_json - ) != identity_payload(recorded_metadata_json) - replay_backfill: BackfillResult = BackfillResult(action=BackfillAction.FORWARD_ONLY) - if compiled_query_hash is not None and fingerprint is not None: - debug_logger: logging.Logger = logging.getLogger("sqlbuild.planner.changes") - query_changed = origin_names is None and detect_query_change( - compiled_query_hash=compiled_query_hash, - fingerprint=fingerprint, - ) - log_debug_event( - logger=debug_logger, - message=( - "fingerprint comparison" - f" compiled_query_hash={compiled_query_hash}" - f" fingerprint_definition_hash={fingerprint.definition_hash}" - f" query_changed={query_changed}" - ), - sqlbuild_subject="model", - sqlbuild_name=model_name, - sqlbuild_event="query_change_check", - sqlbuild_phase="planner", - sqlbuild_status="changed" if query_changed else "unchanged", - ) - log_sql(logger=debug_logger, sql=model.query_sql, action="compiled_query") - log_sql(logger=debug_logger, sql=fingerprint.definition, action="fingerprint_definition") - if query_changed or changed_functions: - replay_backfill = resolve_replay_on_change( - replay_on_change=get_config_str(values=model.config.values, key="replay_on_change") - ) - - schema_findings: tuple[SchemaFinding, ...] = () - schema_backfill: BackfillResult = BackfillResult(action=BackfillAction.FORWARD_ONLY) - warehouse_columns: tuple[ColumnInfo, ...] | None = snapshot.existing_columns.get(model_name) - yml_columns: tuple[ColumnInfo, ...] = _build_yml_columns(model) - inferred_columns: tuple[InferredColumn, ...] | None = model.inferred_columns - has_expected: bool = bool(yml_columns) or ( - inferred_columns is not None and bool(inferred_columns) - ) - type_enforcement: bool = _get_type_enforcement(model) - if warehouse_columns is not None and has_expected: - schema_findings = detect_schema_changes( - yml_columns=yml_columns, - inferred_columns=inferred_columns, - warehouse_columns=warehouse_columns, - type_enforcement=type_enforcement, - inferred_schema_complete=not model.fast_lineage_has_star, - dynamic_columns=( - model.schema_entry.dynamic_columns if model.schema_entry is not None else () - ), - dialect=snapshot.column_dialect, - ) - recorded_columns_hash: str | None = recorded_declared_columns_hash(recorded_metadata_json) - declared_columns_changed: bool = ( - recorded_columns_hash is not None and recorded_columns_hash != current_columns_hash - ) - if schema_findings and ( - query_changed or changed_functions or config_changed or declared_columns_changed - ): - schema_backfill = resolve_replay_on_change( - replay_on_change=get_config_str(values=model.config.values, key="replay_on_change") - ) - - backfill: BackfillResult = pick_more_aggressive(a=replay_backfill, b=schema_backfill) - - change_kind: ChangeKind - if query_changed: - change_kind = ChangeKind.QUERY_CHANGED - elif changed_functions: - change_kind = ChangeKind.FUNCTION_CHANGED - elif schema_findings: - change_kind = ChangeKind.SCHEMA_CHANGED - elif config_changed: - change_kind = ChangeKind.CONFIG_CHANGED - else: - change_kind = ChangeKind.NO_CHANGE - - return ChangeDetectionResult( - model_name=model_name, - change_kind=change_kind, - query_changed=query_changed, - config_changed=config_changed, - fingerprint_metadata_json=metadata_json, - previous_metadata_json=fingerprint.metadata_json if fingerprint is not None else None, - fingerprint_version_hash=expected_version_hash, - previous_version_hash=fingerprint.version_hash if fingerprint is not None else None, - schema_findings=schema_findings, - backfill=backfill, - changed_functions=changed_functions, - ) - - -def _get_type_enforcement(model: CompiledModelView) -> bool: - """Resolve whether type enforcement is active for a model.""" - - if model.schema_entry is not None and model.schema_entry.type_enforcement is not None: - return model.schema_entry.type_enforcement - return False - - -def _build_yml_columns(model: CompiledModelView) -> tuple[ColumnInfo, ...]: - """Build expected columns from schema.yml declarations.""" - - if model.schema_entry is None: - return () - return tuple( - ColumnInfo(name=col.name, type=col.type) - for col in model.schema_entry.columns - if col.type is not None - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py b/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py deleted file mode 100644 index bbc84be95a..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Version identity metadata comparison helpers.""" - -from __future__ import annotations - -import json - -from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY -from sqlbuild.compiler.fingerprints.constants import AUDIT_GATE_METADATA_KEY -from sqlbuild.compiler.planner.constants import ( - DECLARED_COLUMNS_METADATA_KEY, - LOCAL_FUNCTION_HASHES_METADATA_KEY, - MIGRATION_FINGERPRINT_METADATA_KEY, -) - - -def version_identity_metadata_payload(metadata_json: str | None) -> object: - """Return the fingerprint metadata payload that participates in version identity.""" - - if metadata_json is None: - return None - try: - payload: object = json.loads(metadata_json) - except json.JSONDecodeError: - return metadata_json - if not isinstance(payload, dict): - return None - identity_payload: dict[str, object] = dict(payload) - identity_payload.pop(AUDIT_GATE_METADATA_KEY, None) - identity_payload.pop(MIGRATION_FINGERPRINT_METADATA_KEY, None) - identity_payload.pop(DECLARED_COLUMNS_METADATA_KEY, None) - return identity_payload - - -def non_function_identity_metadata_payload(metadata_json: str | None) -> object: - """Return the version identity metadata payload without called-function hashes.""" - - payload: object = version_identity_metadata_payload(metadata_json) - if not isinstance(payload, dict): - return payload - return { - key: value for key, value in payload.items() if key != LOCAL_FUNCTION_HASHES_METADATA_KEY - } - - -def changed_local_function_names( - *, metadata_json: str, previous_metadata_json: str -) -> tuple[str, ...]: - """Return directly called functions whose recorded definition hash changed.""" - - current: dict[str, object] = _local_function_hashes(metadata_json) - previous: dict[str, object] = _local_function_hashes(previous_metadata_json) - return tuple( - sorted( - name - for name, function_hash in current.items() - if name in previous and previous[name] != function_hash - ) - ) - - -def _local_function_hashes(metadata_json: str) -> dict[str, object]: - try: - payload: object = json.loads(metadata_json) - except json.JSONDecodeError: - return {} - if not isinstance(payload, dict): - return {} - hashes: object = payload.get(LOCAL_FUNCTION_HASHES_METADATA_KEY) - return {str(name): value for name, value in hashes.items()} if isinstance(hashes, dict) else {} - - -def with_origin_cursor_input_names(*, metadata_json: str, renamed_refs: dict[str, str]) -> str: - """Return metadata JSON whose cursor_inputs name renamed models by their origin names.""" - - try: - payload: object = json.loads(metadata_json) - except json.JSONDecodeError: - return metadata_json - config: object = payload.get("config") if isinstance(payload, dict) else None - cursor_inputs: object = ( - config.get(CURSOR_INPUTS_CONFIG_KEY) if isinstance(config, dict) else None - ) - if ( - not isinstance(payload, dict) - or not isinstance(config, dict) - or not isinstance(cursor_inputs, dict) - ): - return metadata_json - origin_inputs: dict[str, object] = { - renamed_refs.get(str(name), str(name)): value for name, value in cursor_inputs.items() - } - return json.dumps( - {**payload, "config": {**config, CURSOR_INPUTS_CONFIG_KEY: origin_inputs}}, - sort_keys=True, - separators=(",", ":"), - default=str, - ) - - -def recorded_declared_columns_hash(metadata_json: str | None) -> str | None: - """Return the declared-columns hash a fingerprint row recorded, if it has one.""" - - if metadata_json is None: - return None - try: - payload: object = json.loads(metadata_json) - except json.JSONDecodeError: - return None - value: object = ( - payload.get(DECLARED_COLUMNS_METADATA_KEY) if isinstance(payload, dict) else None - ) - return value if isinstance(value, str) else None - - -def with_declared_columns_hash(*, metadata_json: str, declared_columns_hash: str) -> str: - """Return fingerprint metadata JSON that also records the model's declared-columns hash.""" - - try: - payload: object = json.loads(metadata_json) - except json.JSONDecodeError: - return metadata_json - if not isinstance(payload, dict): - return metadata_json - return json.dumps( - {**payload, DECLARED_COLUMNS_METADATA_KEY: declared_columns_hash}, - sort_keys=True, - separators=(",", ":"), - default=str, - ) - - -def without_declared_columns_hash(metadata_json: str) -> str: - """Return fingerprint metadata JSON without the declared-columns hash key, if present.""" - - try: - payload: object = json.loads(metadata_json) - except json.JSONDecodeError: - return metadata_json - if not isinstance(payload, dict) or DECLARED_COLUMNS_METADATA_KEY not in payload: - return metadata_json - return json.dumps( - {key: value for key, value in payload.items() if key != DECLARED_COLUMNS_METADATA_KEY}, - sort_keys=True, - separators=(",", ":"), - default=str, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/policy.py b/src/sqlbuild/compiler/planner/_helpers/changes/policy.py deleted file mode 100644 index 2195b5de55..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/policy.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Replay-on-change policy resolution.""" - -from __future__ import annotations - -import re - -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import BackfillResult -from sqlbuild.compiler.planner.types import BackfillAction - -_BOUNDED_PATTERN: re.Pattern[str] = re.compile(r"^bounded-(.+)$") - -_ACTION_PRIORITY: dict[BackfillAction, int] = { - BackfillAction.FORWARD_ONLY: 0, - BackfillAction.BOUNDED: 1, - BackfillAction.FULL: 2, -} - - -def resolve_replay_on_change( - *, - replay_on_change: str | None, -) -> BackfillResult: - """Resolve replay behavior for a detected model version change.""" - - return _resolve_replay_value(replay_on_change) - - -def pick_more_aggressive(*, a: BackfillResult, b: BackfillResult) -> BackfillResult: - """Return the more aggressive of two backfill results.""" - - if _ACTION_PRIORITY[a.action] >= _ACTION_PRIORITY[b.action]: - return a - return b - - -def _resolve_replay_value(raw: str | None) -> BackfillResult: - """Parse a raw replay policy value into a BackfillResult.""" - - if raw is None or raw == BackfillAction.FORWARD_ONLY: - return BackfillResult(action=BackfillAction.FORWARD_ONLY) - if raw == BackfillAction.FULL: - return BackfillResult(action=BackfillAction.FULL) - match: re.Match[str] | None = _BOUNDED_PATTERN.match(raw) - if match is not None: - duration: str = match.group(1).strip() - return BackfillResult(action=BackfillAction.BOUNDED, duration=duration) - raise PlannerInputError(f"unknown replay_on_change '{raw}'") diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/query.py b/src/sqlbuild/compiler/planner/_helpers/changes/query.py deleted file mode 100644 index 5cbbb0320f..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/query.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Query change detection by hash comparison.""" - -from __future__ import annotations - -from sqlbuild.compiler.fingerprints.models import Fingerprint - - -def detect_query_change( - *, - compiled_query_hash: str, - fingerprint: Fingerprint, -) -> bool: - """Compare compiled hashes against the stored fingerprint to detect query changes.""" - - return compiled_query_hash != fingerprint.definition_hash diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py b/src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py deleted file mode 100644 index d0d7336477..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Collect the model renames that downstream references may have been rewritten for.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.identity.hashing import model_definition_hash -from sqlbuild.compiler.planner.classes.migration_state_inspection import ( - MigrationStateInspection, -) -from sqlbuild.compiler.planner.models import ( - PlannerRuntime, - PlannerScope, - ReferenceRename, - WarehouseSnapshot, -) -from sqlbuild.compiler.references.types import SqlReferenceKind - - -def plan_reference_renames( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - state: MigrationStateInspection, - schemas: set[str], - planned: dict[str, str], -) -> tuple[ReferenceRename, ...]: - """Return this plan's renames plus recorded renames, read only when a query changed.""" - - if not _has_changed_referencing_query(scope=scope, snapshot=snapshot): - return () - state.inspect_migration_events( - schemas={schema for schema in schemas if schema.lower() in snapshot.migration_state_schemas} - ) - target_name: str | None = runtime.project.effective_target_name - return ( - *( - ReferenceRename(new_name=new_name, origin_name=origin_name) - for new_name, origin_name in planned.items() - if new_name != origin_name - ), - *( - ReferenceRename( - new_name=event.destination_model, - origin_name=event.origin_model, - recorded_at=event.created_at, - ) - for event in state.events - if event.target_name in (None, target_name) - and event.origin_model is not None - and event.origin_model != event.destination_model - ), - ) - - -def _has_changed_referencing_query(*, scope: PlannerScope, snapshot: WarehouseSnapshot) -> bool: - selected: tuple[CompiledModelView, ...] = tuple( - model - for key in scope.selected_keys - if (model := scope.models_by_name.get(key.name)) is not None - ) - return any(_referencing_query_changed(model=model, snapshot=snapshot) for model in selected) - - -def _referencing_query_changed(*, model: CompiledModelView, snapshot: WarehouseSnapshot) -> bool: - recorded: Fingerprint | None = snapshot.fingerprints.models.get(model.name) - return ( - recorded is not None - and any(reference.ref_kind == SqlReferenceKind.REF for reference in model.references) - and model_definition_hash( - model_name=model.name, query_sql=model.query_sql, dialect=snapshot.column_dialect - ) - != recorded.definition_hash - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py b/src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py deleted file mode 100644 index 6560c5b9bd..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Recognize model queries whose only change is references rewritten for renamed models.""" - -from __future__ import annotations - -import re -from datetime import UTC, datetime -from itertools import islice, product - -from sqlbuild._native import compute_query_hash, neutralize_lint_interpolation -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import ReferenceRename - -_REF_CALL: re.Pattern[str] = re.compile( - r"^__ref\s*\(\s*(?P['\"])(?P[^'\"]+)(?P=quote)\s*\)$" -) -_GENERIC_DIALECT: str = "generic" -_MAX_CANDIDATE_MAPPINGS: int = 64 - -type _NameSpan = tuple[int, int, str] - - -def origin_reference_names( - *, - query_sql: str, - recorded: Fingerprint, - renames: tuple[ReferenceRename, ...], - dialect: str | None, -) -> dict[str, str] | None: - """Return the rename mapping under which the query hashes to its recorded definition.""" - - origins: dict[str, list[str]] = {} - rename: ReferenceRename - for rename in renames: - if rename.recorded_at is None or _utc(recorded.ts) < _utc(rename.recorded_at): - origins.setdefault(rename.new_name, []).append(rename.origin_name) - if not origins: - return None - spans: tuple[_NameSpan, ...] = _ref_name_spans( - query_sql=query_sql, dialect=dialect or _GENERIC_DIALECT, names=frozenset(origins) - ) - names: tuple[str, ...] = tuple(sorted({name for _, _, name in spans})) - choices: tuple[tuple[str, ...], ...] = tuple( - _lineage(name=name, origins=origins) for name in names - ) - combination: tuple[str, ...] - for combination in islice(product(*choices), _MAX_CANDIDATE_MAPPINGS): - mapping: dict[str, str] = { - name: origin for name, origin in zip(names, combination, strict=True) if origin != name - } - if mapping and compute_query_hash( - query_sql=_substitute(query_sql=query_sql, spans=spans, mapping=mapping), - dialect=dialect, - ) == (recorded.definition_hash): - return mapping - return None - - -def _ref_name_spans( - *, query_sql: str, dialect: str, names: frozenset[str] -) -> tuple[_NameSpan, ...]: - spans: list[_NameSpan] = [] - original_start: int - original_text: str - for _, _, _, original_start, _, original_text in neutralize_lint_interpolation( - query_sql, dialect - )[1]: - match: re.Match[str] | None = _REF_CALL.match(original_text) - if match is not None and match.group("name") in names: - spans.append( - ( - original_start + match.start("name"), - original_start + match.end("name"), - match.group("name"), - ) - ) - return tuple(spans) - - -def _lineage(*, name: str, origins: dict[str, list[str]]) -> tuple[str, ...]: - seen: list[str] = [name] - index: int = 0 - while index < len(seen): - origin: str - for origin in origins.get(seen[index], ()): - if origin not in seen: - seen.append(origin) - index += 1 - return tuple(seen) - - -def _substitute(*, query_sql: str, spans: tuple[_NameSpan, ...], mapping: dict[str, str]) -> str: - pieces: list[str] = [] - copied_to: int = 0 - start: int - end: int - name: str - for start, end, name in spans: - pieces.extend((query_sql[copied_to:start], mapping.get(name, name))) - copied_to = end - pieces.append(query_sql[copied_to:]) - return "".join(pieces) - - -def _utc(value: datetime) -> datetime: - if value.tzinfo is None or value.utcoffset() is None: - return value.replace(tzinfo=UTC) - return value.astimezone(UTC) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/schema.py b/src/sqlbuild/compiler/planner/_helpers/changes/schema.py deleted file mode 100644 index c563f88afa..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/changes/schema.py +++ /dev/null @@ -1,237 +0,0 @@ -"""Schema change detection by column comparison.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild._native import matching_dynamic_families -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.compile.models import InferredColumn -from sqlbuild.compiler.planner.models import SchemaFinding -from sqlbuild.compiler.planner.types import SchemaChangeKind, SchemaColumnSource -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - -_CASE_FOLDING_DIALECT: str = "snowflake" - - -def detect_schema_changes( - *, - yml_columns: tuple[ColumnInfo, ...], - inferred_columns: tuple[InferredColumn, ...] | None, - warehouse_columns: tuple[ColumnInfo, ...], - type_enforcement: bool, - inferred_schema_complete: bool, - dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = (), - dialect: str | None = None, -) -> tuple[SchemaFinding, ...]: - """Compare yml and inferred columns against warehouse columns and return findings.""" - - if dialect == _CASE_FOLDING_DIALECT: - yml_columns = tuple(replace(col, name=col.name.lower()) for col in yml_columns) - if inferred_columns is not None: - inferred_columns = tuple( - replace(col, name=col.name.lower()) for col in inferred_columns - ) - warehouse_columns = tuple(replace(col, name=col.name.lower()) for col in warehouse_columns) - warehouse_map: dict[str, str] = {col.name: col.type for col in warehouse_columns} - findings: list[SchemaFinding] = [] - - seen_names: set[str] = set() - - if type_enforcement: - findings.extend( - _compare_yml_columns( - yml_columns=yml_columns, warehouse_map=warehouse_map, dialect=dialect - ) - ) - col: ColumnInfo - for col in yml_columns: - seen_names.add(col.name) - - if inferred_columns is not None: - findings.extend( - _compare_inferred_columns( - inferred_columns=inferred_columns, - warehouse_map=warehouse_map, - seen_names=seen_names, - dialect=dialect, - ) - ) - inferred_col: InferredColumn - for inferred_col in inferred_columns: - seen_names.add(inferred_col.name) - else: - if inferred_columns is not None: - findings.extend( - _compare_inferred_columns( - inferred_columns=inferred_columns, - warehouse_map=warehouse_map, - seen_names=seen_names, - dialect=dialect, - ) - ) - inferred_col_ne: InferredColumn - for inferred_col_ne in inferred_columns: - seen_names.add(inferred_col_ne.name) - - findings.extend( - _compare_yml_columns_non_enforced( - yml_columns=yml_columns, - warehouse_map=warehouse_map, - seen_names=seen_names, - dialect=dialect, - ) - ) - col_ne: ColumnInfo - for col_ne in yml_columns: - seen_names.add(col_ne.name) - - for column_name, column_type in warehouse_map.items(): - if column_name in seen_names: - continue - matching: tuple[SchemaDynamicColumnFamily, ...] = matching_dynamic_families( - families=dynamic_columns, - column_name=column_name, - ) - if len(matching) != 1: - continue - family: SchemaDynamicColumnFamily = matching[0] - seen_names.add(column_name) - if not types_equal(left=family.type, right=column_type, dialect=dialect): - findings.append( - SchemaFinding( - kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, - column_name=column_name, - source=SchemaColumnSource.YML, - expected_type=family.type, - actual_type=column_type, - ) - ) - - if type_enforcement or inferred_schema_complete: - col_name: str - col_type: str - for col_name, col_type in warehouse_map.items(): - if col_name not in seen_names: - findings.append( - SchemaFinding( - kind=SchemaChangeKind.COLUMN_REMOVED, - column_name=col_name, - source=( - SchemaColumnSource.YML - if yml_columns - else SchemaColumnSource.SQL_ANALYSIS - ), - actual_type=col_type, - ) - ) - - return tuple(findings) - - -def _compare_yml_columns( - *, - yml_columns: tuple[ColumnInfo, ...], - warehouse_map: dict[str, str], - dialect: str | None, -) -> list[SchemaFinding]: - """Compare yml-declared columns against warehouse state.""" - - findings: list[SchemaFinding] = [] - col: ColumnInfo - for col in yml_columns: - if col.name not in warehouse_map: - findings.append( - SchemaFinding( - kind=SchemaChangeKind.COLUMN_ADDED, - column_name=col.name, - source=SchemaColumnSource.YML, - expected_type=col.type, - ) - ) - elif not types_equal(left=warehouse_map[col.name], right=col.type, dialect=dialect): - findings.append( - SchemaFinding( - kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, - column_name=col.name, - source=SchemaColumnSource.YML, - expected_type=col.type, - actual_type=warehouse_map[col.name], - ) - ) - return findings - - -def _compare_yml_columns_non_enforced( - *, - yml_columns: tuple[ColumnInfo, ...], - warehouse_map: dict[str, str], - seen_names: set[str], - dialect: str | None, -) -> list[SchemaFinding]: - """Compare yml columns against warehouse when type enforcement is off.""" - - findings: list[SchemaFinding] = [] - col: ColumnInfo - for col in yml_columns: - if col.name in seen_names: - continue - if col.name not in warehouse_map: - findings.append( - SchemaFinding( - kind=SchemaChangeKind.COLUMN_ADDED, - column_name=col.name, - source=SchemaColumnSource.YML, - expected_type=col.type, - ) - ) - elif not types_equal(left=warehouse_map[col.name], right=col.type, dialect=dialect): - findings.append( - SchemaFinding( - kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, - column_name=col.name, - source=SchemaColumnSource.YML, - expected_type=col.type, - actual_type=warehouse_map[col.name], - ) - ) - return findings - - -def _compare_inferred_columns( - *, - inferred_columns: tuple[InferredColumn, ...], - warehouse_map: dict[str, str], - seen_names: set[str], - dialect: str | None, -) -> list[SchemaFinding]: - """Compare sql_analysis-inferred columns against warehouse state, skipping yml-covered names.""" - - findings: list[SchemaFinding] = [] - col: InferredColumn - for col in inferred_columns: - if col.name in seen_names: - continue - if col.name not in warehouse_map: - findings.append( - SchemaFinding( - kind=SchemaChangeKind.COLUMN_ADDED, - column_name=col.name, - source=SchemaColumnSource.SQL_ANALYSIS, - expected_type=col.type, - ) - ) - elif col.type is not None and not types_equal( - left=warehouse_map[col.name], right=col.type, dialect=dialect - ): - findings.append( - SchemaFinding( - kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, - column_name=col.name, - source=SchemaColumnSource.SQL_ANALYSIS, - expected_type=col.type, - actual_type=warehouse_map[col.name], - ) - ) - return findings diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py b/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py index 548567bdc4..b60b5bbbc1 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py @@ -2,13 +2,12 @@ from __future__ import annotations -from sqlbuild._native import get_config_str +from sqlbuild._native import detect_model_changes, get_config_str from sqlbuild.adapter.contract.models import ColumnInfo, NormalizedType, RelationInfo from sqlbuild.adapter.contract.types import TypeFamily from sqlbuild.adapter.type_system.main.normalize_type import normalize_type from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.migrations.types import MigrationCompatibility -from sqlbuild.compiler.planner._helpers.changes.detect import detect_model_changes from sqlbuild.compiler.planner.constants import VIEW_RELATION_TYPE_MARKER from sqlbuild.compiler.planner.models import ( ChangeDetectionResult, diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py b/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py index b41baeb9a2..31e74658f5 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py @@ -8,8 +8,8 @@ from collections.abc import Mapping from typing import Any +from sqlbuild._native import version_identity_metadata_payload from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY -from sqlbuild.compiler.planner._helpers.changes.metadata import version_identity_metadata_payload from sqlbuild.compiler.planner.constants import ( MIGRATION_FINGERPRINT_EXCLUDED_CONFIG_KEYS, MIGRATION_LOCAL_NAME_PREFIX, diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index f2cc964f6d..a0df66e45c 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -6,7 +6,7 @@ from dataclasses import replace from datetime import UTC, datetime -from sqlbuild._native import compute_query_hash, get_config_str +from sqlbuild._native import compute_query_hash, get_config_str, plan_reference_renames from sqlbuild.adapter.contract.models import ColumnInfo, MigrationStagePlan, RelationInfo from sqlbuild.adapter.contract.types import BuiltinAdapter, MigrationTransfer from sqlbuild.compiler.compile.constants import MIGRATE_FORCE_CONFIG_KEY, MIGRATE_FROM_CONFIG_KEY @@ -22,7 +22,6 @@ MigrationDiscovery, MigrationPromotion, ) -from sqlbuild.compiler.planner._helpers.changes.recorded_renames import plan_reference_renames from sqlbuild.compiler.planner._helpers.migrations.compatibility import ( MigrationCompatibilityResult, check_migration_compatibility, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index eec6e54e65..1bd31d8da6 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -11,6 +11,7 @@ from sqlbuild._native import cursor_render as render from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild._native import ( + detect_model_changes, get_config_bool, get_config_cursor_bound, get_config_int, @@ -18,6 +19,7 @@ get_config_string_tuple, microbatch_guidance, render_source_relation, + replaces_incremental_relation, resolve_bounded_cursor_override, resolve_cursor_input_roles, resolve_effective_microbatch_batch_size, @@ -96,10 +98,6 @@ ) from sqlbuild.compiler.planner.constants import MICROBATCH_CONCURRENCY_MESSAGE from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.changes._model_changes import detect_model_changes -from sqlbuild.compiler.planner.main.changes.relation_replacement import ( - replaces_incremental_relation, -) from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety from sqlbuild.compiler.planner.main.execution.future_cursor_warning import future_cursor_cap_warning from sqlbuild.compiler.planner.main.execution.maximum_start_warning import maximum_start_cap_warning diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/identities.py b/src/sqlbuild/compiler/planner/_helpers/planning/identities.py index a7e2a38b40..1be7c7f043 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/identities.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/identities.py @@ -4,9 +4,9 @@ from dataclasses import replace +from sqlbuild._native import detect_model_changes_in_scope from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.planner._helpers.changes.detect import detect_model_changes_in_scope from sqlbuild.compiler.planner._helpers.identity.direct import ( build_direct_model_version_identities, ) diff --git a/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py b/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py index 2853d9a560..74d5c886b7 100644 --- a/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py +++ b/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import detect_model_changes from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.fingerprints.models import Fingerprint @@ -9,7 +10,6 @@ classify_selection_staleness_warnings, format_stale_upstream_warning_message, ) -from sqlbuild.compiler.planner.main.changes._model_changes import detect_model_changes from sqlbuild.compiler.planner.models import ( ChangeDetectionResult, DirectModelVersionIdentities, diff --git a/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py b/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py index ddc30ab436..d31d5f9e7e 100644 --- a/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py +++ b/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any, cast -from sqlbuild.compiler.planner._helpers.changes.metadata import without_declared_columns_hash +from sqlbuild._native import without_declared_columns_hash from sqlbuild.compiler.planner._helpers.migrations.fingerprint import ( build_migration_fingerprint, migration_fingerprint_ref_names, diff --git a/src/sqlbuild/compiler/planner/main/changes/_model_changes.py b/src/sqlbuild/compiler/planner/main/changes/_model_changes.py deleted file mode 100644 index d6f30f069b..0000000000 --- a/src/sqlbuild/compiler/planner/main/changes/_model_changes.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Public entrypoint for one-model change detection.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner._helpers.changes.detect import ( - detect_model_changes as _detect_model_changes, -) -from sqlbuild.compiler.planner.models import ChangeDetectionResult, WarehouseSnapshot - - -def detect_model_changes( - *, - model: CompiledModelView, - snapshot: WarehouseSnapshot, - sql_analysis_enabled: bool, - query_change_tracking: bool, - full_refresh: bool, - function_local_hashes: dict[str, str] | None = None, - expected_version_hash: str | None = None, - expected_metadata_json: str | None = None, -) -> ChangeDetectionResult: - """Detect changes for one model and resolve backfill policy.""" - - return _detect_model_changes( - model=model, - snapshot=snapshot, - sql_analysis_enabled=sql_analysis_enabled, - query_change_tracking=query_change_tracking, - full_refresh=full_refresh, - function_local_hashes=function_local_hashes, - expected_version_hash=expected_version_hash, - expected_metadata_json=expected_metadata_json, - ) diff --git a/src/sqlbuild/compiler/planner/main/changes/relation_replacement.py b/src/sqlbuild/compiler/planner/main/changes/relation_replacement.py deleted file mode 100644 index 5d53c18497..0000000000 --- a/src/sqlbuild/compiler/planner/main/changes/relation_replacement.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public incremental relation-replacement predicate.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction - - -def replaces_incremental_relation( - *, materialization_type: MaterializationType, action: PlanAction -) -> bool: - """Identify incremental replacements whose bounds must ignore the destination maximum.""" - - return ( - materialization_type == MaterializationType.INCREMENTAL - and action == PlanAction.CREATE_TABLE - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/execution.py b/src/sqlbuild/compiler/planner/main/execution/execution.py index 12a33f9d6a..d2977a7cb6 100644 --- a/src/sqlbuild/compiler/planner/main/execution/execution.py +++ b/src/sqlbuild/compiler/planner/main/execution/execution.py @@ -7,6 +7,7 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import detect_changes, resolve_model_actions from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.compiler.compile.types import ( @@ -14,8 +15,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner._helpers.changes.actions import resolve_model_actions -from sqlbuild.compiler.planner._helpers.changes.detect import detect_changes from sqlbuild.compiler.planner._helpers.planning.buildability import ( check_selected_scope_buildability, ) diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index 65f2ab648f..1e41b5847a 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -4,14 +4,15 @@ from collections.abc import Callable, Mapping -from sqlbuild._native import assert_no_unresolved_sql_markers, resolve_cursor_intrinsics +from sqlbuild._native import ( + assert_no_unresolved_sql_markers, + replaces_incremental_relation, + resolve_cursor_intrinsics, +) from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL -from sqlbuild.compiler.planner.main.changes.relation_replacement import ( - replaces_incremental_relation, -) from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.compiler.planner.types import ( IncrementalMode, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py b/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py index 70b435d7e3..79b37bd434 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py @@ -4,10 +4,8 @@ from dataclasses import replace +from sqlbuild._native import replaces_incremental_relation from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists -from sqlbuild.compiler.planner.main.changes.relation_replacement import ( - replaces_incremental_relation, -) from sqlbuild.compiler.planner.main.cursor_window.empty_cursor_window import empty_cursor_window from sqlbuild.compiler.planner.main.execution.future_cursor_warning import ( future_cursor_cap_warning, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index 295ba0a2c1..79c0b1d7c9 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -60,7 +60,6 @@ class PlannerChangeDetectionWorkTestCase: expected_model_calls: dict[str, int] expected_warning_count: int query_change_tracking: bool = True - expected_query_tracking_calls: dict[tuple[str, bool], int] = field(default_factory=dict) expected_identity_builds: int = 1 diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py index 323774a72f..fe34b95e93 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py @@ -15,7 +15,7 @@ from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.changes import detect as change_detection +from sqlbuild.compiler.planner._helpers.identity import model_metadata from sqlbuild.compiler.planner._helpers.planning import identities as identity_planning from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( @@ -66,17 +66,12 @@ selection_diagnostics=False, expected_model_calls={"orders": 1}, expected_warning_count=0, - expected_query_tracking_calls={("orders", True): 1}, ), PlannerChangeDetectionWorkTestCase( description="selection diagnostics enabled", selection_diagnostics=True, expected_model_calls={"stg_orders": 1, "orders": 1}, expected_warning_count=0, - expected_query_tracking_calls={ - ("stg_orders", True): 1, - ("orders", True): 1, - }, expected_identity_builds=2, ), PlannerChangeDetectionWorkTestCase( @@ -85,11 +80,6 @@ expected_model_calls={"stg_orders": 1, "orders": 2}, expected_warning_count=0, query_change_tracking=False, - expected_query_tracking_calls={ - ("orders", False): 1, - ("stg_orders", True): 1, - ("orders", True): 1, - }, expected_identity_builds=2, ), ], @@ -129,8 +119,8 @@ def test_given_selected_model_when_planning_then_each_required_model_is_change_d query_change_tracking=test_case.query_change_tracking, ), ) - model_detector: Mock = Mock(wraps=change_detection.detect_model_changes) - monkeypatch.setattr(change_detection, "detect_model_changes", model_detector) + model_detector: Mock = Mock(wraps=model_metadata.declared_columns_hash) + monkeypatch.setattr(model_metadata, "declared_columns_hash", model_detector) identity_builder: Mock = Mock(wraps=identity_planning.build_direct_model_version_identities) monkeypatch.setattr( identity_planning, @@ -150,11 +140,6 @@ def test_given_selected_model_when_planning_then_each_required_model_is_change_d call.kwargs["model"].name for call in model_detector.call_args_list ) assert dict(model_calls) == test_case.expected_model_calls - query_tracking_calls: Counter[tuple[str, bool]] = Counter( - (call.kwargs["model"].name, call.kwargs["query_change_tracking"]) - for call in model_detector.call_args_list - ) - assert dict(query_tracking_calls) == test_case.expected_query_tracking_calls assert identity_builder.call_count == test_case.expected_identity_builds assert len(plan.warnings) == test_case.expected_warning_count diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py index 684c13669f..30a318d01d 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -116,6 +116,7 @@ def test_given_native_source_when_checking_module_paths_then_reports_errors( "sqlbuild.executor.runtime", "sqlbuild.fingerprints", "sqlbuild.lineage", + "sqlbuild.planner.changes", ), ), ], diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index 67848cdf49..c6f621657f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -61,14 +61,6 @@ class DetectFunctionCallerChangeTestCase: expected_backfill: BackfillResult -@dataclass(frozen=True) -class DetectQueryChangeTestCase: - description: str - compiled_query_hash: str - fingerprint_query_hash: str - expected_changed: bool - - @dataclass(frozen=True) class DetectSchemaChangesTestCase: description: str @@ -82,20 +74,6 @@ class DetectSchemaChangesTestCase: dialect: str | None = None -@dataclass(frozen=True) -class ResolveBackfillTestCase: - description: str - raw_value: str | None - expected_result: BackfillResult - - -@dataclass(frozen=True) -class ResolveBackfillErrorTestCase: - description: str - raw_value: str - expected_error_fragment: str - - @dataclass(frozen=True) class DetectRenamedModelTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_declared_columns_metadata.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_declared_columns_metadata.py index ea10347407..9387516453 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_declared_columns_metadata.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_declared_columns_metadata.py @@ -2,10 +2,11 @@ from __future__ import annotations +import json + import pytest -from sqlbuild.compiler.planner._helpers.changes.metadata import ( - recorded_declared_columns_hash, +from sqlbuild._native import ( version_identity_metadata_payload, with_declared_columns_hash, without_declared_columns_hash, @@ -38,7 +39,7 @@ def test_given_fingerprint_metadata_when_reading_declared_columns_hash_then_tolerates_absence( test_case: DeclaredColumnsMetadataTestCase, ) -> None: - assert recorded_declared_columns_hash(test_case.metadata_json) == ( + assert json.loads(test_case.metadata_json).get("declared_columns_hash") == ( test_case.expected_recorded_hash ) assert version_identity_metadata_payload(test_case.metadata_json) == ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 8bdc8b3693..7dd9002187 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -10,7 +10,7 @@ from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.changes.detect import detect_changes, detect_model_changes +from sqlbuild._native import detect_changes, detect_model_changes from sqlbuild.compiler.planner._helpers.output.strategy import ( build_model_warnings, get_materialization_type, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_policy.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_policy.py deleted file mode 100644 index 2b12491a2d..0000000000 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_policy.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.planner._helpers.changes.policy import resolve_replay_on_change -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import BackfillResult -from sqlbuild.compiler.planner.types import BackfillAction -from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( - ResolveBackfillErrorTestCase, - ResolveBackfillTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - ResolveBackfillTestCase( - description="returns full for full policy", - raw_value="full", - expected_result=BackfillResult(action=BackfillAction.FULL), - ), - ResolveBackfillTestCase( - description="returns bounded with duration for bounded policy", - raw_value="bounded-30d", - expected_result=BackfillResult(action=BackfillAction.BOUNDED, duration="30d"), - ), - ResolveBackfillTestCase( - description="returns forward only for null policy", - raw_value=None, - expected_result=BackfillResult(action=BackfillAction.FORWARD_ONLY), - ), - ], - ids=lambda case: case.description, -) -def test_given_policy_when_resolving_replay_on_change_then_returns_expected( - test_case: ResolveBackfillTestCase, -) -> None: - result: BackfillResult = resolve_replay_on_change( - replay_on_change=test_case.raw_value, - ) - - assert result == test_case.expected_result - - -@pytest.mark.parametrize( - "test_case", - [ - ResolveBackfillErrorTestCase( - description="raises for an unvalidated policy value", - raw_value="unknown", - expected_error_fragment="unknown replay_on_change 'unknown'", - ), - ], - ids=lambda case: case.description, -) -def test_given_unvalidated_policy_when_resolving_replay_on_change_then_raises( - test_case: ResolveBackfillErrorTestCase, -) -> None: - with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment): - resolve_replay_on_change(replay_on_change=test_case.raw_value) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_query.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_query.py deleted file mode 100644 index 6321b4b72a..0000000000 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_query.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import annotations - -from datetime import datetime - -import pytest - -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.changes.query import detect_query_change -from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( - DetectQueryChangeTestCase, -) - -_STUB_TS: datetime = datetime(2026, 1, 15, 12, 0, 0) - - -@pytest.mark.parametrize( - "test_case", - [ - DetectQueryChangeTestCase( - description="detects no change when query hashes match", - compiled_query_hash="abc", - fingerprint_query_hash="abc", - expected_changed=False, - ), - DetectQueryChangeTestCase( - description="detects change when query hashes differ", - compiled_query_hash="abc", - fingerprint_query_hash="def", - expected_changed=True, - ), - ], - ids=lambda case: case.description, -) -def test_given_hashes_when_detecting_query_change_then_returns_expected( - test_case: DetectQueryChangeTestCase, -) -> None: - fingerprint: Fingerprint = Fingerprint( - node_type="model", - node_name="test", - target_database=None, - target_schema=None, - target_name="test", - run_id="run_001", - definition_hash=test_case.fingerprint_query_hash, - schema_fingerprint="schema_a", - definition="SELECT 1", - ts=_STUB_TS, - ) - result: bool = detect_query_change( - compiled_query_hash=test_case.compiled_query_hash, - fingerprint=fingerprint, - ) - - assert result == test_case.expected_changed diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py index da4bc9289d..6e9cae5263 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py @@ -6,9 +6,8 @@ import pytest -from sqlbuild._native import compute_query_hash +from sqlbuild._native import compute_query_hash, origin_reference_names from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.changes.reference_renames import origin_reference_names from sqlbuild.compiler.planner.models import ReferenceRename from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( OriginReferenceNamesTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py index d28471cad1..3f39ef5ed8 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py @@ -2,9 +2,10 @@ import pytest +from sqlbuild._native import detect_schema_changes + from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import InferredColumn -from sqlbuild.compiler.planner._helpers.changes.schema import detect_schema_changes from sqlbuild.compiler.planner.models import SchemaFinding from sqlbuild.compiler.planner.types import SchemaChangeKind, SchemaColumnSource from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily From db771384d92c50c47286013c055a1a0710fb49a3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 05:20:39 +0100 Subject: [PATCH 23/40] fix: read dynamic column families from the native extension in change detection --- .../src/bindings/_helpers/planner_changes/detect.rs | 2 +- .../sqlbuild/compiler/planner/_helpers/changes/test_detect.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 372cfe5f50..19c22aa036 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -38,7 +38,7 @@ const HASHING_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.hashin const FUNCTIONS_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.functions"; const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; const TYPES_EQUAL_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; -const DYNAMIC_FAMILIES_MODULE: &str = "sqlbuild.spec.contracts.main.matching_dynamic_families"; +const DYNAMIC_FAMILIES_MODULE: &str = "sqlbuild._native"; const DEBUG_EVENT_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; const LOG_SQL_MODULE: &str = "sqlbuild.diagnostics.main.log_sql"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 7dd9002187..c07c411f1b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -5,12 +5,11 @@ import pytest -from sqlbuild._native import compute_query_hash +from sqlbuild._native import compute_query_hash, detect_changes, detect_model_changes from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild._native import detect_changes, detect_model_changes from sqlbuild.compiler.planner._helpers.output.strategy import ( build_model_warnings, get_materialization_type, From 4be5439c13007ee701423d46d4581c69c3b20f00 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 05:20:45 +0100 Subject: [PATCH 24/40] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 1183af3b8b..ddf4206d32 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 57868 +remaining_lines = 56777 [[stays]] path = "executor" From c3aa0413867f0921b22e6da07ec8c3bb2ee053a5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 05:20:52 +0100 Subject: [PATCH 25/40] style: sort imports in planner change tests --- .../sqlbuild/compiler/planner/_helpers/changes/test_schema.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py index 3f39ef5ed8..e99f28e530 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py @@ -3,7 +3,6 @@ import pytest from sqlbuild._native import detect_schema_changes - from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.planner.models import SchemaFinding From b9e0daa0c4b22e290da739d1448f44f33fc22a00 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 04:54:53 +0100 Subject: [PATCH 26/40] chore: keep user-code and public API modules as Python --- python-boundary.toml | 200 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) diff --git a/python-boundary.toml b/python-boundary.toml index ddf4206d32..c23af68ebc 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -99,6 +99,16 @@ path = "diagnostics/classes/log_record_route_filter.py" reason = "public-api" note = "A logging.Filter that routes records to destinations." +[[stays]] +path = "diagnostics/main" +reason = "public-api" +note = "Diagnostics entry points built on Python logging: configuration, context, debug and SQL logging." + +[[stays]] +path = "diagnostics/_helpers/process_resources.py" +reason = "public-api" +note = "Samples process resources for debug logging through Python's resource module." + [[stays]] path = "rule_engine/_helpers/host" reason = "user-code" @@ -124,6 +134,106 @@ path = "rule_engine/classes/rule_context.py" reason = "user-code" note = "The context object user rules receive." +[[stays]] +path = "rule_engine/_helpers/engine/definition.py" +reason = "user-code" +note = "The @rule decorator implementation; inspects user rule functions and their signatures." + +[[stays]] +path = "rule_engine/_helpers/engine/hermeticity.py" +reason = "user-code" +note = "Statically checks the imports of user rule source files." + +[[stays]] +path = "rule_engine/_helpers/engine/rule_harness.py" +reason = "user-code" +note = "Public custom-rule test harness that runs user rules over a real compiled project." + +[[stays]] +path = "rule_engine/_helpers/engine/fact_replay.py" +reason = "user-code" +note = "Recomputes the fact-view values a custom rule read, for its cache key." + +[[stays]] +path = "rule_engine/main/define.py" +reason = "user-code" +note = "Public @rule decorator entry point." + +[[stays]] +path = "rule_engine/main/evaluate_rule.py" +reason = "user-code" +note = "Public custom-rule test harness entry point." + +[[stays]] +path = "rule_engine/main/_evaluate.py" +reason = "user-code" +note = "Project evaluation entry used by the public custom-rule test harness." + +[[stays]] +path = "rule_engine/classes/audit_facts.py" +reason = "user-code" +note = "Fact view user rules read through RuleContext; the Python API over native data." + +[[stays]] +path = "rule_engine/classes/column_facts.py" +reason = "user-code" +note = "Fact view user rules read through RuleContext; the Python API over native data." + +[[stays]] +path = "rule_engine/classes/contract_facts.py" +reason = "user-code" +note = "Fact view user rules read through RuleContext; the Python API over native data." + +[[stays]] +path = "rule_engine/classes/declaration_facts.py" +reason = "user-code" +note = "Fact view user rules read through RuleContext; the Python API over native data." + +[[stays]] +path = "rule_engine/classes/graph_facts.py" +reason = "user-code" +note = "Fact view user rules read through RuleContext; the Python API over native data." + +[[stays]] +path = "rule_engine/classes/model_test_facts.py" +reason = "user-code" +note = "Fact view user rules read through RuleContext; the Python API over native data." + +[[stays]] +path = "rule_engine/classes/project_facts.py" +reason = "user-code" +note = "Fact view user rules read through RuleContext; the Python API over native data." + +[[stays]] +path = "rule_engine/classes/sql_facts.py" +reason = "user-code" +note = "Fact view user rules read through RuleContext; the Python API over native data." + +[[stays]] +path = "rule_engine/classes/project_tree.py" +reason = "user-code" +note = "Project-tree fact view user rules read through RuleContext." + +[[stays]] +path = "rule_engine/classes/sql_document.py" +reason = "user-code" +note = "Lazy Polyglot SQL document user rules read through RuleContext." + +[[stays]] +path = "rule_engine/classes/fact_reads.py" +reason = "user-code" +note = "Records the fact-view reads of one user rule evaluation." + +[[stays]] +path = "rule_engine/classes/fact_digests.py" +reason = "user-code" +note = "Digests the fact-view values user rules read, for the custom-rule cache." + +[[stays]] +path = "rule_engine/classes/shared_fact_encoder.py" +reason = "user-code" +note = "Encodes the Python fact-view values user rules read, for their digests." + [[stays]] path = "runtime/observability" reason = "public-api" @@ -239,6 +349,96 @@ path = "compiler/pipeline/_helpers/materializations.py" reason = "user-code" note = "Imports user materialization modules." +[[stays]] +path = "compiler/frontier/_helpers/code_identity.py" +reason = "user-code" +note = "Fingerprints the loaded Python modules behind cached compiler and rule results." + +[[stays]] +path = "compiler/frontier/main/compiled_code_identity.py" +reason = "user-code" +note = "Entry point for the loaded-code identity of cached compiler and rule results." + +[[stays]] +path = "compiler/macro_bridge/main" +reason = "user-code" +note = "Macro bridge entry points for the running render stage that executes user macros." + +[[stays]] +path = "compiler/compile/_helpers/macro_bridge/call_store.py" +reason = "user-code" +note = "Hands the running macro bridge its user macro call store." + +[[stays]] +path = "compiler/python_nodes/main/identity.py" +reason = "user-code" +note = "Identity entry point over live user node functions." + +[[stays]] +path = "compiler/python_nodes/main/_hook_identities.py" +reason = "user-code" +note = "Builds identities of loaded user hook functions." + +[[stays]] +path = "compiler/python_nodes/main/_graph.py" +reason = "user-code" +note = "Builds the Python node graph from loaded user functions." + +[[stays]] +path = "compiler/python_nodes/main/_run_selection.py" +reason = "user-code" +note = "Builds the Python node graph from loaded user functions for run selection." + +[[stays]] +path = "compiler/auditing/main/above.py" +reason = "public-api" +note = "Audit bound helper project audit factories call." + +[[stays]] +path = "compiler/auditing/main/below.py" +reason = "public-api" +note = "Audit bound helper project audit factories call." + +[[stays]] +path = "compiler/auditing/main/outside.py" +reason = "public-api" +note = "Audit bound helper project audit factories call." + +[[stays]] +path = "compiler/auditing/main/audit_factory.py" +reason = "public-api" +note = "Decorator that declares project audit factories." + +[[stays]] +path = "compiler/auditing/main/get_audit_factory_definition.py" +reason = "public-api" +note = "Reads the declaration attached to a project audit factory." + +[[stays]] +path = "compiler/auditing/main/_builtins.py" +reason = "public-api" +note = "Built-in audit factories written against the public audit factory API." + +[[stays]] +path = "compiler/sql_analysis/main/import_polyglot.py" +reason = "public-api" +note = "Imports the Polyglot Python package for the Python API." + +[[stays]] +path = "compiler/sql_analysis/main/import_polyglot_sql.py" +reason = "public-api" +note = "Imports the Polyglot SQL Python package for the Python API." + +[[stays]] +path = "compiler/sql_analysis/_helpers/polyglot.py" +reason = "public-api" +note = "Wraps the Polyglot Python package with complexity guards." + +[[stays]] +path = "compiler/profiling" +reason = "public-api" +note = "Compile timing collectors built on Python context managers and clocks." + [[stays]] path = "assets.py" reason = "public-api" From 73a52e8de514cb3e5e64186ae323647660611107 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 04:55:30 +0100 Subject: [PATCH 27/40] refactor: delete the unused rule SQL AST helpers --- src/sqlbuild/rule_engine/_helpers/sql/ast.py | 92 -------------------- src/sqlbuild/rule_engine/constants.py | 2 - 2 files changed, 94 deletions(-) delete mode 100644 src/sqlbuild/rule_engine/_helpers/sql/ast.py diff --git a/src/sqlbuild/rule_engine/_helpers/sql/ast.py b/src/sqlbuild/rule_engine/_helpers/sql/ast.py deleted file mode 100644 index a24a0b4ff4..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/sql/ast.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Polyglot AST traversal and payload helpers for rules.""" - -from __future__ import annotations - -from collections.abc import Iterator -from typing import Any, cast - -from sqlbuild.rule_engine.constants import AST_SELECT_KIND, NAMED_CTE_TUPLE_SIZE - - -def payload(node: Any) -> dict[str, object]: - """Return the single expression payload from a Polyglot node.""" - - raw: object = node.to_dict() - if not isinstance(raw, dict) or len(raw) != 1: - return {} - value: object = next(iter(raw.values())) - return cast(dict[str, object], value) if isinstance(value, dict) else {} - - -def kind(node: Any) -> str: - """Return a stable lowercase Polyglot node kind.""" - - return str(getattr(node, "key", "") or getattr(node, "kind", "")).lower() - - -def nodes(*, root: Any, wanted: str) -> Iterator[Any]: - """Yield every node whose stable kind matches wanted.""" - - for node in root.walk(): - if kind(node) == wanted: - yield node - - -def name_value(value: object) -> str: - """Extract a name from Polyglot's nested name payload.""" - - if isinstance(value, str): - return value - if isinstance(value, dict): - mapping: dict[str, object] = cast(dict[str, object], value) - raw: object = mapping.get("name") - if isinstance(raw, str): - return raw - if isinstance(raw, dict): - return name_value(raw) - return "" - - -def cte_name(node: Any) -> str: - if isinstance(node, tuple) and len(node) == NAMED_CTE_TUPLE_SIZE: - return str(node[0]) - return str(getattr(node, "alias_or_name", "") or getattr(node, "name", "")) - - -def top_level_ctes(root: Any) -> tuple[Any, ...]: - """Return top-level CTE nodes in authored order.""" - - data: dict[str, object] = payload(root) - with_payload: object = data.get("with") - raw_ctes: object = ( - cast(dict[str, object], with_payload).get("ctes") - if isinstance(with_payload, dict) - else None - ) - if not isinstance(raw_ctes, list): - return () - candidates: tuple[Any, ...] = tuple(root.walk()) - matched_ids: set[int] = set() - ctes: list[tuple[str, Any]] = [] - for raw_cte in raw_ctes: - if not isinstance(raw_cte, dict): - return () - cte_mapping: dict[str, object] = cast(dict[str, object], raw_cte) - alias: str = name_value(cte_mapping.get("alias")) - raw_body: object = cte_mapping.get("this") - body: Any | None = None - for candidate in candidates: - if id(candidate) in matched_ids or candidate is root: - continue - if candidate.to_dict() == raw_body: - body = candidate - matched_ids.add(id(candidate)) - break - if not alias or body is None: - return () - ctes.append((alias, body)) - return tuple(ctes) - - -def select_payload(node: Any) -> dict[str, object]: - return payload(node) if kind(node) == AST_SELECT_KIND else {} diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index d985125218..b388d81a9b 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -76,7 +76,6 @@ SKILL_STALE: str = "stale" REPLACEABLE_SKILL_STATES: frozenset[str] = frozenset({SKILL_FRESH, SKILL_MISSING, SKILL_STALE}) -AST_SELECT_KIND: str = "select" AST_CTE_KIND: str = "cte" CONTRACT_ENFORCED: str = "enforced" TYPE_PROOF_RULE_CODES: frozenset[str] = frozenset({"SQBRCONTRACT105"}) @@ -86,7 +85,6 @@ TIMESTAMP_TYPE: str = "TIMESTAMP" -NAMED_CTE_TUPLE_SIZE: int = 2 TARGET_DIRECTORY_NAME: str = "target" RULE_DECORATOR_TOKEN: str = "@rule" INIT_MODULE_NAME: str = "__init__" From eff40b10188054a277e34f6c3322e5ee046332b8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 05:25:02 +0100 Subject: [PATCH 28/40] refactor: call the native scope bindings directly and delete the scope wrappers --- .../compile_attachment/declaration_scope.rs | 25 ++- .../_helpers/refactor/refactor_facts.rs | 32 ++-- .../_helpers/render/declaration_values.rs | 7 +- .../bindings/_helpers/scopes/scope_command.rs | 95 ++++++++--- .../bindings/_helpers/scopes/scope_index.rs | 32 +++- .../bindings/_helpers/scopes/scope_loading.rs | 13 +- .../bindings/_helpers/scopes/scope_lookup.rs | 24 +-- .../bindings/_helpers/scopes/scope_reports.rs | 153 +++++++++++++++--- .../_helpers/scopes/scope_validation.rs | 46 +++++- .../_helpers/scopes/scope_visibility.rs | 33 +++- fensu.toml | 5 + src/sqlbuild/_native.pyi | 92 ++++++----- .../main/_scope_index_with_compile_usages.py | 4 +- .../compiler/scopes/_helpers/__init__.py | 1 - .../compiler/scopes/_helpers/paths.py | 17 -- src/sqlbuild/compiler/scopes/main/__init__.py | 1 - .../scopes/main/_build_scope_index.py | 25 --- .../scopes/main/_declaration_lexical_path.py | 18 --- .../_get_placement_validated_scope_index.py | 17 -- .../scopes/main/_open_native_scope_index.py | 20 --- .../scopes/main/_query_scope_target.py | 21 --- .../_resolve_scope_declaration_visibility.py | 16 -- .../main/_resolve_scope_path_visibility.py | 16 -- .../scopes/main/_resolve_scope_visibility.py | 16 -- .../scopes/main/_validate_scope_index.py | 17 -- .../scopes/main/browse_scope_folders.py | 18 --- .../scopes/main/build_scope_lookup.py | 12 -- .../scopes/main/explain_scope_declaration.py | 27 ---- .../scopes/main/list_scope_declarations.py | 24 --- .../scopes/main/preview_scope_move.py | 19 --- .../scopes/main/query_scope_report.py | 27 ---- .../main/relocate_declarations_for_move.py | 14 -- .../compiler/scopes/main/scope_metadata.py | 13 -- .../scopes/main/serialize_scope_report.py | 10 -- .../rule_engine/_helpers/run/native_rows.py | 3 +- .../sqlbuild/compiler/scopes/_test_types.py | 8 - .../src/sqlbuild/compiler/scopes/helpers.py | 42 +---- .../scopes/test_native_scope_parity.py | 21 --- .../cli/commands/_helpers/scope/helpers.py | 4 +- .../compiler/compile/_helpers/helpers.py | 2 +- .../_helpers/test_declaration_scopes.py | 8 +- .../sqlbuild/compiler/scopes/_test_helpers.py | 16 +- .../src/sqlbuild/compiler/scopes/helpers.py | 8 +- .../scopes/test_builder_and_visibility.py | 12 +- .../compiler/scopes/test_identities.py | 4 +- .../scopes/test_lookup_and_metadata.py | 3 +- .../sqlbuild/compiler/scopes/test_paths.py | 6 +- .../compiler/scopes/test_placement.py | 6 +- .../compiler/scopes/test_relocation.py | 2 +- .../sqlbuild/compiler/scopes/test_reports.py | 16 +- .../_helpers/engine/test_scope_facts.py | 2 +- 51 files changed, 475 insertions(+), 598 deletions(-) delete mode 100644 src/sqlbuild/compiler/scopes/_helpers/__init__.py delete mode 100644 src/sqlbuild/compiler/scopes/_helpers/paths.py delete mode 100644 src/sqlbuild/compiler/scopes/main/__init__.py delete mode 100644 src/sqlbuild/compiler/scopes/main/_build_scope_index.py delete mode 100644 src/sqlbuild/compiler/scopes/main/_declaration_lexical_path.py delete mode 100644 src/sqlbuild/compiler/scopes/main/_get_placement_validated_scope_index.py delete mode 100644 src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py delete mode 100644 src/sqlbuild/compiler/scopes/main/_query_scope_target.py delete mode 100644 src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py delete mode 100644 src/sqlbuild/compiler/scopes/main/_resolve_scope_path_visibility.py delete mode 100644 src/sqlbuild/compiler/scopes/main/_resolve_scope_visibility.py delete mode 100644 src/sqlbuild/compiler/scopes/main/_validate_scope_index.py delete mode 100644 src/sqlbuild/compiler/scopes/main/browse_scope_folders.py delete mode 100644 src/sqlbuild/compiler/scopes/main/build_scope_lookup.py delete mode 100644 src/sqlbuild/compiler/scopes/main/explain_scope_declaration.py delete mode 100644 src/sqlbuild/compiler/scopes/main/list_scope_declarations.py delete mode 100644 src/sqlbuild/compiler/scopes/main/preview_scope_move.py delete mode 100644 src/sqlbuild/compiler/scopes/main/query_scope_report.py delete mode 100644 src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py delete mode 100644 src/sqlbuild/compiler/scopes/main/scope_metadata.py delete mode 100644 src/sqlbuild/compiler/scopes/main/serialize_scope_report.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs index cce2deebf4..f3769dc70d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -6,6 +6,9 @@ use pyo3::types::PyTuple; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::render::declaration_values::declaration_scope_resolver; +use crate::bindings::_helpers::scopes::scope_index::open_native_scope_index; +use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; +use crate::bindings::_helpers::scopes::scope_validation::validate_scope_index; const NATIVE_MODULE: &str = "sqlbuild._native"; const SCAN_CACHE_MODULE: &str = "sqlbuild.compiler.compile.classes.sql_test_scan_cache"; @@ -13,8 +16,6 @@ const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; -const OPEN_INDEX_MODULE: &str = "sqlbuild.compiler.scopes.main._open_native_scope_index"; -const VALIDATE_INDEX_MODULE: &str = "sqlbuild.compiler.scopes.main._validate_scope_index"; fn compile_error(py: Python<'_>, message: Bound<'_, PyAny>) -> PyResult { Ok(PyErr::from_value( @@ -66,17 +67,12 @@ pub(crate) fn build_declaration_scope<'py>( &scenario_files, sql_lexical_syntax.getattr("native_mapping")?, ))?; - let native: Bound<'py, PyAny> = call_with( - &imported(py, OPEN_INDEX_MODULE, "open_native_scope_index")?, - &[ - ("discovered_inputs", discovered_inputs.clone()), - ("loaded_macros", loaded_macros.clone()), - ], - )?; - let validated: PyResult> = call_with( - &imported(py, VALIDATE_INDEX_MODULE, "validate_scope_index")?, - &[("index", native.getattr("index")?)], - ); + let native: Bound<'py, PyAny> = Bound::new( + py, + open_native_scope_index(py, discovered_inputs, loaded_macros, None)?, + )? + .into_any(); + let validated: Result<(), PyErr> = validate_scope_index(&native.getattr("index")?); if let Err(error) = validated { if !error.is_instance( py, @@ -120,8 +116,7 @@ pub(crate) fn build_declaration_scope<'py>( } } let index: Bound<'py, PyAny> = native.call_method0("index_with_relationships")?; - let lookup: Bound<'py, PyAny> = - imported(py, NATIVE_MODULE, "build_scope_lookup")?.call1((&index,))?; + let lookup: Bound<'py, PyAny> = build_scope_lookup(py, &index)?.into_bound(py); scope_build( discovered_inputs, loaded_macros, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs index 18b161d1d6..205b8f1d28 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs @@ -2,8 +2,8 @@ use std::collections::HashMap; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use sqlbuild_refactor::refactoring::models::{ AuthoredFile, DeclarationPlacement, DependencyFacts, DestinationFacts, DiscoveredFile, ExpansionFacts, ExpansionSpan, LocationFacts, ManualLocation, ModelFacts, PlacedDeclaration, @@ -11,6 +11,10 @@ use sqlbuild_refactor::refactoring::models::{ SchemaColumnFacts, SqlFileRole, }; +use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; +use crate::bindings::_helpers::scopes::scope_reports::scope_move_preview; +use crate::bindings::_helpers::scopes::scope_validation::scope_relocated_declarations; + const NATIVE_MODULE: &str = "sqlbuild._native"; const CONFIG_MODULE: &str = "sqlbuild._native"; const SCOPE_MODELS: &str = "sqlbuild.compiler.scopes.models"; @@ -317,11 +321,14 @@ pub(crate) fn declaration_placement( .getattr("graph")? .getattr("project")? .getattr("scope_index")?; - let lookup: Bound<'_, PyAny> = native(py, "build_scope_lookup")?.call1((&index,))?; - let (preview, diagnostics): (Bound<'_, PyAny>, Bound<'_, PyAny>) = - native(py, "scope_move_preview")? - .call1((lookup, format!("{MODEL_KIND_PREFIX}{model}"), destination))? - .extract()?; + let lookup: Bound<'_, PyAny> = build_scope_lookup(py, &index)?.into_bound(py); + let (preview, diagnostics) = scope_move_preview( + py, + &lookup, + &PyString::new(py, &format!("{MODEL_KIND_PREFIX}{model}")).into_any(), + &PyString::new(py, destination).into_any(), + )?; + let diagnostics: Bound<'_, PyAny> = diagnostics.into_bound(py); let relocated: Option> = if preview.is_none() { None } else { @@ -337,11 +344,10 @@ pub(crate) fn declaration_placement( .import(SCOPE_MODELS)? .getattr("ResourceIdentity")? .call((), Some(&options))?; - let records: Bound<'_, PyAny> = - native(py, "scope_relocated_declarations")?.call1((&index, resource, destination))?; - if records.is_none() { - None - } else { + let records: Option> = + scope_relocated_declarations(py, &index, &resource, destination)?; + if let Some(records) = records { + let records: Bound<'_, PyAny> = records.into_bound(py).into_any(); let sort: Bound<'_, PyDict> = PyDict::new(py); sort.set_item( "key", @@ -363,6 +369,8 @@ pub(crate) fn declaration_placement( }); } Some(relocated) + } else { + None } }; let mut declarations: Vec = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs index 3cd5fa6bd4..d3972c62b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs @@ -7,12 +7,12 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyInt, PyList, PyString, P use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; -const SCOPE_LOOKUP_MODULE: &str = "sqlbuild.compiler.scopes.main.build_scope_lookup"; const NATIVE_MODULE: &str = "sqlbuild._native"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; @@ -133,10 +133,7 @@ pub(crate) fn declaration_scope_resolver<'py>( let py: Python<'py> = discovered.py(); let declarations: Bound<'py, PyDict> = runtime_declarations(discovered, loaded_macros)?; let lookup: Bound<'py, PyAny> = if lookup.is_none() { - call_with( - &imported(py, SCOPE_LOOKUP_MODULE, "build_scope_lookup")?, - &[("index", scope_index.clone())], - )? + build_scope_lookup(py, scope_index)?.into_bound(py) } else { lookup.clone() }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs index bfef737d93..2087649059 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs @@ -5,7 +5,7 @@ use std::collections::BTreeSet; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_scopes::scope_report::models::ScopeResult; @@ -13,8 +13,13 @@ use sqlbuild_scopes::scope_text::main::render_scope_text::render_scope_text; use sqlbuild_scopes::scope_text::main::scope_result_failed::scope_result_failed; use sqlbuild_scopes::scope_text::models::ScopeTextRequest; +use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; use crate::bindings::_helpers::scopes::scope_objects::ScopeClasses; use crate::bindings::_helpers::scopes::scope_report_objects::read_result; +use crate::bindings::_helpers::scopes::scope_reports::{ + scope_browse, scope_explain, scope_list, scope_move_preview, scope_report, + serialize_scope_report, +}; const NATIVE_MODULE: &str = "sqlbuild._native"; const MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; @@ -218,40 +223,74 @@ fn merged<'py>( replaced(report, &changes) } +/// `value` as an optional Python string. +fn optional_text<'py>(py: Python<'py>, value: &Option) -> Option> { + value + .as_deref() + .map(|text| PyString::new(py, text).into_any()) +} + /// Run the query `request` selects over `lookup`. fn query_result<'py>( py: Python<'py>, lookup: &Bound<'py, PyAny>, (request, target): (&Bound<'py, PyAny>, Option), ) -> PyResult> { - let native: Bound<'_, PyModule> = py.import(NATIVE_MODULE)?; let at: Option = request.getattr("at")?.extract()?; let prospective: bool = at.is_some(); let browse: Option = request.getattr("browse")?.extract()?; if let Some(folder) = browse { - return native - .getattr("scope_browse")? - .call1((lookup, (folder, &target, prospective))); + return Ok(scope_browse( + py, + lookup, + (folder, optional_text(py, &target), prospective), + )? + .into_bound(py)); } let filters: Bound<'_, PyAny> = report_filters(py, request)?; let list_path: Option = request.getattr("list_path")?.extract()?; if let Some(folder) = list_path { - return native - .getattr("scope_list")? - .call1((lookup, (folder, &filters, &target, prospective))); + return Ok(scope_list( + py, + lookup, + ( + folder, + Some(filters), + optional_text(py, &target), + prospective, + ), + )? + .into_bound(py)); } let directory: bool = at .as_deref() .is_some_and(|at| at.ends_with('/') || at.ends_with('\\')); - let mut report: Bound<'_, PyAny> = native - .getattr("scope_report")? - .call1((lookup, (&target, &at, directory, &filters)))?; + let mut report: Bound<'_, PyAny> = scope_report( + py, + lookup, + ( + optional_text(py, &target), + optional_text(py, &at), + directory, + Some(filters), + ), + )? + .into_bound(py); let explain: Option = request.getattr("explain")?.extract()?; if let Some(declaration) = explain { - let (explanation, diagnostics): (Bound<'_, PyAny>, Bound<'_, PyAny>) = native - .getattr("scope_explain")? - .call1((lookup, (declaration, &target, &at, directory)))? - .extract()?; + let (explanation, diagnostics): (Bound<'_, PyAny>, Bound<'_, PyAny>) = scope_explain( + py, + lookup, + ( + PyString::new(py, &declaration).into_any(), + optional_text(py, &target), + optional_text(py, &at), + directory, + ), + ) + .map(|(explanation, diagnostics)| { + (explanation.into_bound(py), diagnostics.into_bound(py)) + })?; let complete: bool = explanation.getattr("complete")?.extract()?; report = merged( &report, @@ -261,10 +300,21 @@ fn query_result<'py>( } let as_path: Option = request.getattr("as_path")?.extract()?; if let Some(destination) = as_path { - let (preview, diagnostics): (Bound<'_, PyAny>, Bound<'_, PyAny>) = native - .getattr("scope_move_preview")? - .call1((lookup, target.clone().unwrap_or_default(), destination))? - .extract()?; + let (preview, diagnostics): (Bound<'_, PyAny>, Bound<'_, PyAny>) = scope_move_preview( + py, + lookup, + &PyString::new(py, &target.clone().unwrap_or_default()).into_any(), + &PyString::new(py, &destination).into_any(), + ) + .map(|(preview, diagnostics)| { + ( + preview.map_or_else( + || py.None().into_bound(py), + |preview| preview.into_bound(py), + ), + diagnostics.into_bound(py), + ) + })?; let complete: bool = !preview.is_none() && preview.getattr("complete")?.extract()?; report = merged(&report, ("move_preview", preview), (diagnostics, complete))?; } @@ -311,17 +361,14 @@ fn run_scope_command( ], )?; let index: Bound<'_, PyAny> = loader.call((), Some(&options))?; - let lookup: Bound<'_, PyAny> = native.getattr("build_scope_lookup")?.call1((index,))?; + let lookup: Bound<'_, PyAny> = build_scope_lookup(py, &index)?.into_bound(py); let target: Option = qualify_target(py, &lookup, request.getattr("target")?.extract()?)?; let result: Bound<'_, PyAny> = query_result(py, &lookup, (request, target.clone()))?; let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; let (output, native_result): (String, ScopeResult) = if request.getattr("json_output")?.extract()? { - let output: String = native - .getattr("serialize_scope_report")? - .call1((&result,))? - .extract()?; + let output: String = serialize_scope_report(py, &result)?; (output, read_result(&classes, &result)?) } else { let native_result: ScopeResult = read_result(&classes, &result)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs index 0aa58ddfae..d72e06f2ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs @@ -897,12 +897,18 @@ impl NativeScopeIndex { /// Python's `open_native_scope_index`: the natively built index of discovered inputs. #[pyfunction] -fn open_native_scope_index( +#[pyo3(signature = (*, discovered_inputs, loaded_macros, static_macros = None))] +pub(crate) fn open_native_scope_index( py: Python<'_>, discovered_inputs: &Bound<'_, PyAny>, loaded_macros: &Bound<'_, PyAny>, - static_macros: &Bound<'_, PyAny>, + static_macros: Option>, ) -> PyResult { + let static_macros: Bound<'_, PyAny> = match static_macros { + Some(static_macros) => static_macros, + None => PyTuple::empty(py).into_any(), + }; + let static_macros: &Bound<'_, PyAny> = &static_macros; let mut walk: Walk<'_> = Walk::new(py)?; walk.add_models(discovered_inputs)?; walk.add_resources(discovered_inputs)?; @@ -922,8 +928,30 @@ fn open_native_scope_index( materialise(walk, native) } +/// Python's `build_scope_index`: the natively built, partially valid index of discovered inputs. +#[pyfunction] +#[pyo3( + name = "build_scope_index", + signature = (*, discovered_inputs, loaded_macros = None, static_macros = None) +)] +fn build_scope_index_binding( + py: Python<'_>, + discovered_inputs: &Bound<'_, PyAny>, + loaded_macros: Option>, + static_macros: Option>, +) -> PyResult> { + let loaded_macros: Bound<'_, PyAny> = match loaded_macros { + Some(loaded_macros) if !loaded_macros.is_none() => loaded_macros, + _ => PyDict::new(py).into_any(), + }; + let native: NativeScopeIndex = + open_native_scope_index(py, discovered_inputs, &loaded_macros, static_macros)?; + Ok(native.index.clone_ref(py)) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_function(wrap_pyfunction!(open_native_scope_index, module)?)?; + module.add_function(wrap_pyfunction!(build_scope_index_binding, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index 6500eb0a10..ec270ed060 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -14,6 +14,7 @@ use crate::bindings::_helpers::discovery::scope_snapshot::discover_scope_snapsho use crate::bindings::_helpers::scopes::scope_cache::{ project_fingerprint, read_scope_index_cache, write_scope_index_cache, }; +use crate::bindings::_helpers::scopes::scope_index::open_native_scope_index; use crate::bindings::_helpers::scopes::scope_tolerant::scope_tolerant_index; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -150,8 +151,16 @@ fn tolerant_index<'py>( &imported(py, MACRO_INVENTORY_MODULE, "inventory_project_macros")?, &[("macro_files", discovered.getattr("macro_files")?)], )?; - let native: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "open_native_scope_index")? - .call1((&discovered, PyDict::new(py), inventory.getattr("exports")?))?; + let native: Bound<'py, PyAny> = Bound::new( + py, + open_native_scope_index( + py, + &discovered, + &PyDict::new(py).into_any(), + Some(inventory.getattr("exports")?), + )?, + )? + .into_any(); let scan_store: Bound<'py, PyTuple> = PyTuple::new( py, [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs index f6dd0d5a72..a93b2a472f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs @@ -269,7 +269,8 @@ pub(crate) fn scope_path_error(py: Python<'_>, message: String) -> PyResult, index: &Bound<'_, PyAny>) -> PyResult> { +#[pyo3(signature = (*, index))] +pub(crate) fn build_scope_lookup(py: Python<'_>, index: &Bound<'_, PyAny>) -> PyResult> { let records: ReportRecords = report_records(&read_index(index)?); let built = py .compiler_detach(|| Ok(report_lookup(&records))) @@ -347,18 +348,17 @@ fn build_scope_lookup(py: Python<'_>, index: &Bound<'_, PyAny>) -> PyResult (Option, Option) { - match normalize_scope_path(path) { - Ok(normalized) => (Some(normalized), None), - Err(failure) => ( - None, - Some(path_message(&RejectedPath { - failure, - path: path.to_owned(), - })), - ), +#[pyo3(name = "normalize_scope_path", signature = (*, path))] +fn scope_normalize_path(py: Python<'_>, path: &Bound<'_, PyAny>) -> PyResult { + let path: String = path.str()?.extract()?; + match normalize_scope_path(&path) { + Ok(normalized) => Ok(normalized), + Err(failure) => Err(scope_path_error( + py, + path_message(&RejectedPath { failure, path }), + )?), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs index 211ba545f0..b4dc0d9265 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs @@ -4,6 +4,7 @@ use pyo3::exceptions::{PyIndexError, PyValueError}; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; +use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::json::models::JsonValue; use sqlbuild_scopes::scope_report::main::browse_scope_folders::browse_scope_folders; @@ -19,6 +20,7 @@ use sqlbuild_scopes::scope_report::models::{ ScopeResult, }; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::scopes::scope_lookup::{ NativeScopeLookup, identity_message, path_message, }; @@ -35,21 +37,21 @@ use crate::bindings::_helpers::scopes::scope_report_objects::{ type PyObject = Py; /// Target, location, whether it is a directory, and the filters. -type ReportArguments<'py> = ( +pub(crate) type ReportArguments<'py> = ( Option>, Option>, bool, Option>, ); /// Declaration, target, location and whether it is a directory. -type ExplainArguments<'py> = ( +pub(crate) type ExplainArguments<'py> = ( Bound<'py, PyAny>, Option>, Option>, bool, ); /// Folder, filters, target and whether the target is prospective. -type ListArguments<'py> = ( +pub(crate) type ListArguments<'py> = ( String, Option>, Option>, @@ -57,6 +59,7 @@ type ListArguments<'py> = ( ); const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const CURRENT_FOLDER: &str = "."; /// One lookup's native lookup and every declaration's values in input order. struct LookupReports { @@ -141,8 +144,7 @@ fn raised(diagnostics: Vec) -> Vec { } /// `query_scope_report`: one existing target or prospective path; invalid paths raise. -#[pyfunction] -fn scope_report( +pub(crate) fn scope_report( py: Python<'_>, lookup: &Bound<'_, PyAny>, request: ReportArguments<'_>, @@ -171,8 +173,7 @@ fn scope_report( } /// `explain_scope_declaration`: one qualified declaration in an optional context. -#[pyfunction] -fn scope_explain( +pub(crate) fn scope_explain( py: Python<'_>, lookup: &Bound<'_, PyAny>, request: ExplainArguments<'_>, @@ -218,8 +219,7 @@ fn folder_request( } /// `browse_scope_folders`: the direct child declaration-definition folders of `folder`. -#[pyfunction] -fn scope_browse( +pub(crate) fn scope_browse( py: Python<'_>, lookup: &Bound<'_, PyAny>, request: (String, Option>, bool), @@ -234,8 +234,7 @@ fn scope_browse( } /// `list_scope_declarations`: declarations recursively beneath one definition folder. -#[pyfunction] -fn scope_list( +pub(crate) fn scope_list( py: Python<'_>, lookup: &Bound<'_, PyAny>, request: ListArguments<'_>, @@ -257,7 +256,8 @@ fn scope_list( /// `preview_scope_move`: visibility deltas of a resource move, without mutating the lookup. #[pyfunction] -fn scope_move_preview( +#[pyo3(name = "preview_scope_move", signature = (*, lookup, resource, destination))] +pub(crate) fn scope_move_preview( py: Python<'_>, lookup: &Bound<'_, PyAny>, resource: &Bound<'_, PyAny>, @@ -295,7 +295,11 @@ fn serialized(classes: &ScopeClasses<'_>, result: &Bound<'_, PyAny>) -> PyResult /// `serialize_scope_report`: deterministic schema-version-two ASCII JSON of one result. #[pyfunction] -fn serialize_scope_report(py: Python<'_>, report: &Bound<'_, PyAny>) -> PyResult { +#[pyo3(signature = (*, report))] +pub(crate) fn serialize_scope_report( + py: Python<'_>, + report: &Bound<'_, PyAny>, +) -> PyResult { let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; match serialized(&classes, report) { Err(error) if error.is_instance_of::(py) => { @@ -310,18 +314,131 @@ fn serialize_scope_report(py: Python<'_>, report: &Bound<'_, PyAny>) -> PyResult /// `scope_metadata_projection` of one `ScopeIndex`, as the dictionary Python builds. #[pyfunction] -#[pyo3(name = "scope_metadata_projection")] +#[pyo3(name = "scope_metadata_projection", signature = (*, index))] fn metadata_projection(py: Python<'_>, index: &Bound<'_, PyAny>) -> PyResult { let index: ProjectionIndex = projection_index(read_index(index)?); let projection: JsonValue = scope_metadata_projection(&index); json_to_python(py, projection, false) } +fn optional<'py>( + arguments: &KeywordArguments<'py>, + name: &str, +) -> PyResult>> { + Ok(Some(arguments.value(name)?).filter(|value| !value.is_none())) +} + +/// Python's `query_scope_report`. +#[pyfunction] +#[pyo3(name = "query_scope_report", signature = (**options))] +fn query_scope_report_binding<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "query_scope_report", + options, + (&["lookup"], &["target", "at", "directory", "filters"]), + )?; + scope_report( + py, + &arguments.value("lookup")?, + ( + optional(&arguments, "target")?, + optional(&arguments, "at")?, + arguments.flag("directory")?, + optional(&arguments, "filters")?, + ), + ) +} + +/// Python's `explain_scope_declaration`. +#[pyfunction] +#[pyo3(name = "explain_scope_declaration", signature = (**options))] +fn explain_scope_declaration_binding<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult<(PyObject, PyObject)> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "explain_scope_declaration", + options, + (&["lookup", "declaration"], &["target", "at", "directory"]), + )?; + scope_explain( + py, + &arguments.value("lookup")?, + ( + arguments.value("declaration")?, + optional(&arguments, "target")?, + optional(&arguments, "at")?, + arguments.flag("directory")?, + ), + ) +} + +/// Python's `browse_scope_folders`. +#[pyfunction] +#[pyo3(name = "browse_scope_folders", signature = (**options))] +fn browse_scope_folders_binding<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "browse_scope_folders", + options, + (&["lookup"], &["folder", "target", "target_is_prospective"]), + )?; + let folder: String = match optional(&arguments, "folder")? { + Some(folder) => folder.extract()?, + None => CURRENT_FOLDER.to_owned(), + }; + scope_browse( + py, + &arguments.value("lookup")?, + ( + folder, + optional(&arguments, "target")?, + arguments.flag("target_is_prospective")?, + ), + ) +} + +/// Python's `list_scope_declarations`. +#[pyfunction] +#[pyo3(name = "list_scope_declarations", signature = (**options))] +fn list_scope_declarations_binding<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "list_scope_declarations", + options, + ( + &["lookup", "folder"], + &["filters", "target", "target_is_prospective"], + ), + )?; + scope_list( + py, + &arguments.value("lookup")?, + ( + arguments.value("folder")?.extract()?, + optional(&arguments, "filters")?, + optional(&arguments, "target")?, + arguments.flag("target_is_prospective")?, + ), + ) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scope_report, module)?)?; - module.add_function(wrap_pyfunction!(scope_explain, module)?)?; - module.add_function(wrap_pyfunction!(scope_browse, module)?)?; - module.add_function(wrap_pyfunction!(scope_list, module)?)?; + module.add_function(wrap_pyfunction!(query_scope_report_binding, module)?)?; + module.add_function(wrap_pyfunction!(explain_scope_declaration_binding, module)?)?; + module.add_function(wrap_pyfunction!(browse_scope_folders_binding, module)?)?; + module.add_function(wrap_pyfunction!(list_scope_declarations_binding, module)?)?; module.add_function(wrap_pyfunction!(scope_move_preview, module)?)?; module.add_function(wrap_pyfunction!(serialize_scope_report, module)?)?; module.add_function(wrap_pyfunction!(metadata_projection, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs index 2ffafd3ecf..fc76ea4a16 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs @@ -1,6 +1,8 @@ //! Compile usage merging, placement validation and move relocation over a Python `ScopeIndex`. -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_scopes::placement::main::diagnostic_order::diagnostic_order; @@ -23,6 +25,8 @@ use crate::bindings::_helpers::scopes::scope_objects::{ use crate::bindings::types::CompilerDetach; const DATACLASSES_MODULE: &str = "dataclasses"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; const RUNTIME_USAGE: usize = 2; /// `dataclasses.replace(item, **changes)`. @@ -249,8 +253,11 @@ fn placement_validated_index<'py>( /// Complete usage and placement diagnostics, merging any collected compile usages first. #[pyfunction] -#[pyo3(signature = (index, enforce_placement, compile_usages))] -fn scope_placement_validated_index( +#[pyo3( + name = "get_placement_validated_scope_index", + signature = (*, index, enforce_placement = true, compile_usages = None) +)] +pub(crate) fn scope_placement_validated_index( index: &Bound<'_, PyAny>, enforce_placement: bool, compile_usages: Option>, @@ -264,7 +271,11 @@ fn scope_placement_validated_index( /// Declarations whose placement changes when `resource` moves, or `None` if unsettled. #[pyfunction] -fn scope_relocated_declarations( +#[pyo3( + name = "relocate_declarations_for_move", + signature = (*, index, resource, destination) +)] +pub(crate) fn scope_relocated_declarations( py: Python<'_>, index: &Bound<'_, PyAny>, resource: &Bound<'_, PyAny>, @@ -308,7 +319,34 @@ fn scope_relocated_declarations( Ok(Some(PyTuple::new(py, moved)?.unbind())) } +/// Python's `validate_scope_index`: raise one `ScopeValidationError` for its error diagnostics. +#[pyfunction] +#[pyo3(signature = (*, index))] +pub(crate) fn validate_scope_index(index: &Bound<'_, PyAny>) -> Result<(), PyErr> { + let py: Python<'_> = index.py(); + let error: Bound<'_, PyAny> = py + .import(SCOPE_TYPES_MODULE)? + .getattr("DiagnosticSeverity")? + .getattr("ERROR")?; + let mut errors: Vec> = Vec::new(); + for item in index.getattr("diagnostics")?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + if item.getattr("severity")?.is(&error) { + errors.push(item); + } + } + if errors.is_empty() { + return Ok(()); + } + Err(PyErr::from_value( + py.import(SCOPE_ERRORS_MODULE)? + .getattr("ScopeValidationError")? + .call1((PyTuple::new(py, errors)?,))?, + )) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(validate_scope_index, module)?)?; module.add_function(wrap_pyfunction!(scope_placement_validated_index, module)?)?; module.add_function(wrap_pyfunction!(scope_relocated_declarations, module)?)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs index 20f038e1f5..91425ea463 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs @@ -92,6 +92,7 @@ fn target_query<'py>( /// `query_target`: the `ScopeTargetQuery` of `target`; invalid text raises the scope error. #[pyfunction] +#[pyo3(name = "query_scope_target", signature = (*, lookup, target))] fn scope_query_target( py: Python<'_>, lookup: &Bound<'_, PyAny>, @@ -112,9 +113,30 @@ fn resolved( Ok(rows) } -/// Every visible fact of each matched resource, and its inaccessible declarations. +/// Python's `resolve_scope_visibility`: visible facts and inaccessible reasons per target. +#[pyfunction] +#[pyo3(signature = (*, lookup, target))] +fn resolve_scope_visibility( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + target: &Bound<'_, PyAny>, +) -> PyResult { + resolved_visibility(py, lookup, target, true) +} + +/// Python's `resolve_scope_declaration_visibility`: visible facts and inaccessible identities. #[pyfunction] -fn scope_resolve_visibility( +#[pyo3(signature = (*, lookup, target))] +fn resolve_scope_declaration_visibility( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + target: &Bound<'_, PyAny>, +) -> PyResult { + resolved_visibility(py, lookup, target, false) +} + +/// Every visible fact of each matched resource, and its inaccessible declarations. +fn resolved_visibility( py: Python<'_>, lookup: &Bound<'_, PyAny>, target: &Bound<'_, PyAny>, @@ -177,6 +199,7 @@ fn scope_resolve_visibility( /// Visible and inaccessible declarations for an authored path, including declaration files. #[pyfunction] +#[pyo3(name = "resolve_scope_path_visibility", signature = (*, lookup, path))] fn scope_resolve_path_visibility( py: Python<'_>, lookup: &Bound<'_, PyAny>, @@ -233,7 +256,11 @@ fn scope_resolve_path_visibility( pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(scope_query_target, module)?)?; - module.add_function(wrap_pyfunction!(scope_resolve_visibility, module)?)?; + module.add_function(wrap_pyfunction!(resolve_scope_visibility, module)?)?; + module.add_function(wrap_pyfunction!( + resolve_scope_declaration_visibility, + module + )?)?; module.add_function(wrap_pyfunction!(scope_resolve_path_visibility, module)?)?; Ok(()) } diff --git a/fensu.toml b/fensu.toml index d0485850c4..005136f41a 100644 --- a/fensu.toml +++ b/fensu.toml @@ -93,6 +93,11 @@ rule = "FFR309" path = "src/sqlbuild/compiler/refactoring/__init__.py" reason = "Refactorings run natively; the package keeps the compiled-project input, manual-location model and error classes the native command reads and raises." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/scopes/__init__.py" +reason = "Declaration scopes are built, queried and validated natively; the package keeps the index, lookup and report models and the errors the native bindings raise." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/sql_test_glue/__init__.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 789ed7c1cb..8656b3341a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5,7 +5,7 @@ from contextlib import AbstractContextManager from contextvars import Token from datetime import date, datetime from pathlib import Path, PurePath -from typing import Any, Literal, TextIO, TypedDict, overload +from typing import Any, TextIO, TypedDict from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter @@ -2419,64 +2419,76 @@ class NativeScopeIndex: def index_with_relationships(self) -> ScopeIndex: ... def open_native_scope_index( + *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: Mapping[str, LoadedMacro], - static_macros: tuple[StaticMacroExport, ...], - /, + static_macros: tuple[StaticMacroExport, ...] = (), ) -> NativeScopeIndex: ... -def scope_placement_validated_index( +def build_scope_index( + *, + discovered_inputs: DiscoveredProjectInputs, + loaded_macros: Mapping[str, LoadedMacro] | None = None, + static_macros: tuple[StaticMacroExport, ...] = (), +) -> ScopeIndex: ... +def get_placement_validated_scope_index( + *, index: ScopeIndex, - enforce_placement: bool, - compile_usages: tuple[UsageRecord, ...] | None, + enforce_placement: bool = True, + compile_usages: tuple[UsageRecord, ...] | None = None, ) -> ScopeIndex: ... -def scope_relocated_declarations( - index: ScopeIndex, resource: ResourceIdentity, destination: str +def relocate_declarations_for_move( + *, index: ScopeIndex, resource: ResourceIdentity, destination: str ) -> tuple[DeclarationRecord, ...] | None: ... +def validate_scope_index(*, index: ScopeIndex) -> None: ... class NativeScopeLookup: ... -def scope_query_target(lookup: ScopeLookup, target: ScopeQueryTarget, /) -> ScopeTargetQuery: ... -@overload -def scope_resolve_visibility( - lookup: ScopeLookup, target: ScopeQueryTarget, with_reasons: Literal[True], / +def query_scope_target(*, lookup: ScopeLookup, target: ScopeQueryTarget) -> ScopeTargetQuery: ... +def resolve_scope_visibility( + *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath ) -> VisibilityResolution: ... -@overload -def scope_resolve_visibility( - lookup: ScopeLookup, target: ScopeQueryTarget, with_reasons: Literal[False], / +def resolve_scope_declaration_visibility( + *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath ) -> DeclarationVisibility: ... -def scope_resolve_path_visibility( - lookup: ScopeLookup, path: str | PurePath, / +def resolve_scope_path_visibility( + *, lookup: ScopeLookup, path: str | PurePath ) -> tuple[tuple[DeclarationRecord, ...], tuple[DeclarationRecord, ...]]: ... -def build_scope_lookup(index: ScopeIndex) -> ScopeLookup: ... -def scope_report( +def build_scope_lookup(*, index: ScopeIndex) -> ScopeLookup: ... +def query_scope_report( + *, lookup: ScopeLookup, - request: tuple[ScopeQueryTarget | None, str | PurePath | None, bool, ScopeReportFilters | None], - /, + target: str | PurePath | ResourceIdentity | DeclarationIdentity | None = None, + at: str | PurePath | None = None, + directory: bool = False, + filters: ScopeReportFilters | None = None, ) -> ScopeReport: ... -def scope_explain( +def explain_scope_declaration( + *, lookup: ScopeLookup, - request: tuple[ - str | DeclarationIdentity, - str | PurePath | ResourceIdentity | None, - str | PurePath | None, - bool, - ], - /, + declaration: str | DeclarationIdentity, + target: str | PurePath | ResourceIdentity | None = None, + at: str | PurePath | None = None, + directory: bool = False, ) -> tuple[DeclarationExplanation, tuple[ScopeDiagnostic, ...]]: ... -def scope_browse( +def browse_scope_folders( + *, lookup: ScopeLookup, - request: tuple[str, str | PurePath | ResourceIdentity | None, bool], - /, + folder: str = ".", + target: str | PurePath | ResourceIdentity | None = None, + target_is_prospective: bool = False, ) -> ScopeBrowseResult: ... -def scope_list( +def list_scope_declarations( + *, lookup: ScopeLookup, - request: tuple[str, ScopeReportFilters | None, str | PurePath | ResourceIdentity | None, bool], - /, + folder: str, + filters: ScopeReportFilters | None = None, + target: str | PurePath | ResourceIdentity | None = None, + target_is_prospective: bool = False, ) -> ScopeListResult: ... -def scope_move_preview( - lookup: ScopeLookup, resource: str | ResourceIdentity, destination: str | PurePath +def preview_scope_move( + *, lookup: ScopeLookup, resource: str | ResourceIdentity, destination: str | PurePath ) -> tuple[MovePreview | None, tuple[ScopeDiagnostic, ...]]: ... -def serialize_scope_report(report: ScopeReport | ScopeBrowseResult | ScopeListResult) -> str: ... +def serialize_scope_report(*, report: ScopeReport | ScopeBrowseResult | ScopeListResult) -> str: ... def run_scope_command( *, request: ScopeCommandRequest, @@ -2489,7 +2501,7 @@ def render_scope_result( request: ScopeCommandRequest, use_color: bool = False, ) -> str: ... -def scope_metadata_projection(index: ScopeIndex) -> dict[str, JsonValue]: ... +def scope_metadata_projection(*, index: ScopeIndex) -> dict[str, JsonValue]: ... def effective_config_target_name( project_config: object, local_config: object, selected: str | None, / ) -> tuple[str | None, str | None]: ... @@ -2660,7 +2672,7 @@ def validate_local_sql_analysis(project_dir: Path, flags: tuple[bool, bool, bool def sql_test_cte_span( contents: str, sql_body: str, cte_name: str, call: str | None, / ) -> tuple[int, int, int, int]: ... -def scope_normalize_path(path: str, /) -> tuple[str | None, str | None]: ... +def normalize_scope_path(*, path: str | PurePath) -> str: ... def scope_index_fingerprint(*, project_dir: Path) -> str | None: ... def load_or_build_scope_index(*, project_dir: Path, no_cache: bool = False) -> ScopeIndex: ... def model_schema_scope_usages( diff --git a/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py b/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py index 8ccdcc776a..b19902120c 100644 --- a/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py +++ b/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py @@ -3,10 +3,10 @@ from __future__ import annotations import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.scopes.main._get_placement_validated_scope_index import ( +from sqlbuild._native import ( get_placement_validated_scope_index, ) +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.scopes.models import ScopeIndex, UsageRecord diff --git a/src/sqlbuild/compiler/scopes/_helpers/__init__.py b/src/sqlbuild/compiler/scopes/_helpers/__init__.py deleted file mode 100644 index 8b6d28cf22..0000000000 --- a/src/sqlbuild/compiler/scopes/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Internal compiler scope helpers.""" diff --git a/src/sqlbuild/compiler/scopes/_helpers/paths.py b/src/sqlbuild/compiler/scopes/_helpers/paths.py deleted file mode 100644 index b63d0b00e8..0000000000 --- a/src/sqlbuild/compiler/scopes/_helpers/paths.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Project-relative scope path normalization, answered by the native scope paths.""" - -from __future__ import annotations - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError - - -def normalize_path(*, path: str | PurePath) -> str: - """The canonical project-relative POSIX path; absolute or escaping paths raise.""" - - normalized, error = _native.scope_normalize_path(str(path)) - if normalized is None: - raise InvalidScopePathError(error or "") - return normalized diff --git a/src/sqlbuild/compiler/scopes/main/__init__.py b/src/sqlbuild/compiler/scopes/main/__init__.py deleted file mode 100644 index 97dd363eac..0000000000 --- a/src/sqlbuild/compiler/scopes/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compiler scope entry modules.""" diff --git a/src/sqlbuild/compiler/scopes/main/_build_scope_index.py b/src/sqlbuild/compiler/scopes/main/_build_scope_index.py deleted file mode 100644 index c1a3187392..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_build_scope_index.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Build the canonical static scope index from discovered project inputs.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.compiler.compile.models import LoadedMacro, StaticMacroExport -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index -from sqlbuild.compiler.scopes.models import ScopeIndex - - -def build_scope_index( - *, - discovered_inputs: DiscoveredProjectInputs, - loaded_macros: Mapping[str, LoadedMacro] | None = None, - static_macros: tuple[StaticMacroExport, ...] = (), -) -> ScopeIndex: - """Project discovery facts into a deterministic, partially valid scope index.""" - - return open_native_scope_index( - discovered_inputs=discovered_inputs, - loaded_macros=loaded_macros or {}, - static_macros=static_macros, - ).index diff --git a/src/sqlbuild/compiler/scopes/main/_declaration_lexical_path.py b/src/sqlbuild/compiler/scopes/main/_declaration_lexical_path.py deleted file mode 100644 index 7856678e28..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_declaration_lexical_path.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Lexical location of a declaration file for visibility resolution.""" - -from __future__ import annotations - -from sqlbuild.compiler.scopes.constants import ( - CURRENT_PATH_COMPONENT, - DECLARATION_LEXICAL_STEM, - PATH_SEPARATOR, -) -from sqlbuild.compiler.scopes.models import DeclarationRecord - - -def declaration_lexical_path(*, record: DeclarationRecord) -> str: - """Return the authored path whose folder controls what a declaration file can use.""" - - return ( - f"{record.owning_path or CURRENT_PATH_COMPONENT}{PATH_SEPARATOR}{DECLARATION_LEXICAL_STEM}" - ) diff --git a/src/sqlbuild/compiler/scopes/main/_get_placement_validated_scope_index.py b/src/sqlbuild/compiler/scopes/main/_get_placement_validated_scope_index.py deleted file mode 100644 index 08cd685100..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_get_placement_validated_scope_index.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Public entrypoint for declaration placement validation.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ScopeIndex, UsageRecord - - -def get_placement_validated_scope_index( - *, - index: ScopeIndex, - enforce_placement: bool = True, - compile_usages: tuple[UsageRecord, ...] | None = None, -) -> ScopeIndex: - """Return complete usage and placement diagnostics, merging any collected compile usages.""" - - return _native.scope_placement_validated_index(index, enforce_placement, compile_usages) diff --git a/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py b/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py deleted file mode 100644 index 2679b5344f..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Build the static scope index natively.""" - -from __future__ import annotations - -from collections.abc import Mapping - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import LoadedMacro, StaticMacroExport -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs - - -def open_native_scope_index( - *, - discovered_inputs: DiscoveredProjectInputs, - loaded_macros: Mapping[str, LoadedMacro], - static_macros: tuple[StaticMacroExport, ...] = (), -) -> _native.NativeScopeIndex: - """Return the natively built index; discovery's normalized paths always index.""" - - return _native.open_native_scope_index(discovered_inputs, loaded_macros, static_macros) diff --git a/src/sqlbuild/compiler/scopes/main/_query_scope_target.py b/src/sqlbuild/compiler/scopes/main/_query_scope_target.py deleted file mode 100644 index d1ab0243c3..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_query_scope_target.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Resolve what one scope target names.""" - -from __future__ import annotations - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - ResourceIdentity, - ScopeLookup, - ScopeTargetQuery, -) - - -def query_scope_target( - *, lookup: ScopeLookup, target: ResourceIdentity | DeclarationIdentity | str | PurePath -) -> ScopeTargetQuery: - """The resources or declarations `target` names; invalid text raises the scope error.""" - - return _native.scope_query_target(lookup, target) diff --git a/src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py b/src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py deleted file mode 100644 index 427b4814f6..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Resolve static declaration visibility without per-declaration inaccessible reasons.""" - -from __future__ import annotations - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import DeclarationVisibility, ResourceIdentity, ScopeLookup - - -def resolve_scope_declaration_visibility( - *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath -) -> DeclarationVisibility: - """Return visible facts and inaccessible declaration identities for one target.""" - - return _native.scope_resolve_visibility(lookup, target, False) diff --git a/src/sqlbuild/compiler/scopes/main/_resolve_scope_path_visibility.py b/src/sqlbuild/compiler/scopes/main/_resolve_scope_path_visibility.py deleted file mode 100644 index acb0dbf959..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_resolve_scope_path_visibility.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Resolve declaration visibility for any authored project-relative path.""" - -from __future__ import annotations - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import DeclarationRecord, ScopeLookup - - -def resolve_scope_path_visibility( - *, lookup: ScopeLookup, path: str | PurePath -) -> tuple[tuple[DeclarationRecord, ...], tuple[DeclarationRecord, ...]]: - """Return visible and inaccessible declarations for an authored path.""" - - return _native.scope_resolve_path_visibility(lookup, path) diff --git a/src/sqlbuild/compiler/scopes/main/_resolve_scope_visibility.py b/src/sqlbuild/compiler/scopes/main/_resolve_scope_visibility.py deleted file mode 100644 index 098231c4dc..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_resolve_scope_visibility.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Resolve canonical static declaration visibility for one target.""" - -from __future__ import annotations - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ResourceIdentity, ScopeLookup, VisibilityResolution - - -def resolve_scope_visibility( - *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath -) -> VisibilityResolution: - """Return visible and inaccessible facts, or an explicit unknown target result.""" - - return _native.scope_resolve_visibility(lookup, target, True) diff --git a/src/sqlbuild/compiler/scopes/main/_validate_scope_index.py b/src/sqlbuild/compiler/scopes/main/_validate_scope_index.py deleted file mode 100644 index 2052f0b25c..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_validate_scope_index.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Strict validation for an already-built, fact-preserving scope index.""" - -from __future__ import annotations - -from sqlbuild.compiler.scopes.exceptions import ScopeValidationError -from sqlbuild.compiler.scopes.models import ScopeDiagnostic, ScopeIndex -from sqlbuild.compiler.scopes.types import DiagnosticSeverity - - -def validate_scope_index(*, index: ScopeIndex) -> None: - """Raise one aggregate compiler error when the index has error diagnostics.""" - - errors: tuple[ScopeDiagnostic, ...] = tuple( - item for item in index.diagnostics if item.severity is DiagnosticSeverity.ERROR - ) - if errors: - raise ScopeValidationError(errors) diff --git a/src/sqlbuild/compiler/scopes/main/browse_scope_folders.py b/src/sqlbuild/compiler/scopes/main/browse_scope_folders.py deleted file mode 100644 index c91b4f0f26..0000000000 --- a/src/sqlbuild/compiler/scopes/main/browse_scope_folders.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Public declaration folder browse operation.""" - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ResourceIdentity, ScopeBrowseResult, ScopeLookup - - -def browse_scope_folders( - *, - lookup: ScopeLookup, - folder: str = ".", - target: str | PurePath | ResourceIdentity | None = None, - target_is_prospective: bool = False, -) -> ScopeBrowseResult: - """Browse direct declaration-definition child folders only.""" - - return _native.scope_browse(lookup, (folder, target, target_is_prospective)) diff --git a/src/sqlbuild/compiler/scopes/main/build_scope_lookup.py b/src/sqlbuild/compiler/scopes/main/build_scope_lookup.py deleted file mode 100644 index d96e802679..0000000000 --- a/src/sqlbuild/compiler/scopes/main/build_scope_lookup.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Build immutable lookup indexes over canonical scope facts.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ScopeIndex, ScopeLookup - - -def build_scope_lookup(*, index: ScopeIndex) -> ScopeLookup: - """Canonicalize scope facts and build deeply immutable lookup mappings.""" - - return _native.build_scope_lookup(index) diff --git a/src/sqlbuild/compiler/scopes/main/explain_scope_declaration.py b/src/sqlbuild/compiler/scopes/main/explain_scope_declaration.py deleted file mode 100644 index f2f339fa40..0000000000 --- a/src/sqlbuild/compiler/scopes/main/explain_scope_declaration.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Public detailed declaration explanation operation.""" - -from __future__ import annotations - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ( - DeclarationExplanation, - DeclarationIdentity, - ResourceIdentity, - ScopeDiagnostic, - ScopeLookup, -) - - -def explain_scope_declaration( - *, - lookup: ScopeLookup, - declaration: str | DeclarationIdentity, - target: str | PurePath | ResourceIdentity | None = None, - at: str | PurePath | None = None, - directory: bool = False, -) -> tuple[DeclarationExplanation, tuple[ScopeDiagnostic, ...]]: - """Explain a qualified declaration in an optional resource/path context.""" - - return _native.scope_explain(lookup, (declaration, target, at, directory)) diff --git a/src/sqlbuild/compiler/scopes/main/list_scope_declarations.py b/src/sqlbuild/compiler/scopes/main/list_scope_declarations.py deleted file mode 100644 index e46c688f92..0000000000 --- a/src/sqlbuild/compiler/scopes/main/list_scope_declarations.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Public recursive declaration folder list operation.""" - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ( - ResourceIdentity, - ScopeListResult, - ScopeLookup, - ScopeReportFilters, -) - - -def list_scope_declarations( - *, - lookup: ScopeLookup, - folder: str, - filters: ScopeReportFilters | None = None, - target: str | PurePath | ResourceIdentity | None = None, - target_is_prospective: bool = False, -) -> ScopeListResult: - """List declarations recursively beneath one browsed folder.""" - - return _native.scope_list(lookup, (folder, filters, target, target_is_prospective)) diff --git a/src/sqlbuild/compiler/scopes/main/preview_scope_move.py b/src/sqlbuild/compiler/scopes/main/preview_scope_move.py deleted file mode 100644 index 6b4c3a999c..0000000000 --- a/src/sqlbuild/compiler/scopes/main/preview_scope_move.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Public pure resource move preview operation.""" - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ( - MovePreview, - ResourceIdentity, - ScopeDiagnostic, - ScopeLookup, -) - - -def preview_scope_move( - *, lookup: ScopeLookup, resource: str | ResourceIdentity, destination: str | PurePath -) -> tuple[MovePreview | None, tuple[ScopeDiagnostic, ...]]: - """Preview declaration visibility changes without mutating compiler facts.""" - - return _native.scope_move_preview(lookup, resource, destination) diff --git a/src/sqlbuild/compiler/scopes/main/query_scope_report.py b/src/sqlbuild/compiler/scopes/main/query_scope_report.py deleted file mode 100644 index 59ccc704a8..0000000000 --- a/src/sqlbuild/compiler/scopes/main/query_scope_report.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Public pure scope report operation.""" - -from __future__ import annotations - -from pathlib import PurePath - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - ResourceIdentity, - ScopeLookup, - ScopeReport, - ScopeReportFilters, -) - - -def query_scope_report( - *, - lookup: ScopeLookup, - target: str | PurePath | ResourceIdentity | DeclarationIdentity | None = None, - at: str | PurePath | None = None, - directory: bool = False, - filters: ScopeReportFilters | None = None, -) -> ScopeReport: - """Query immutable compiler-owned scope facts without filesystem access.""" - - return _native.scope_report(lookup, (target, at, directory, filters)) diff --git a/src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py b/src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py deleted file mode 100644 index a962bc5632..0000000000 --- a/src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public declaration relocation for a resource move.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import DeclarationRecord, ResourceIdentity, ScopeIndex - - -def relocate_declarations_for_move( - *, index: ScopeIndex, resource: ResourceIdentity, destination: str -) -> tuple[DeclarationRecord, ...] | None: - """Return declarations that must move, at their new paths, when a resource moves.""" - - return _native.scope_relocated_declarations(index, resource, destination) diff --git a/src/sqlbuild/compiler/scopes/main/scope_metadata.py b/src/sqlbuild/compiler/scopes/main/scope_metadata.py deleted file mode 100644 index c6af02e388..0000000000 --- a/src/sqlbuild/compiler/scopes/main/scope_metadata.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public value-safe projection and serialization of canonical compiler scope facts.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ScopeIndex -from sqlbuild.compiler.scopes.types import JsonValue - - -def scope_metadata_projection(*, index: ScopeIndex) -> dict[str, JsonValue]: - """Return deterministic, value-free scope metadata for external consumers.""" - - return _native.scope_metadata_projection(index) diff --git a/src/sqlbuild/compiler/scopes/main/serialize_scope_report.py b/src/sqlbuild/compiler/scopes/main/serialize_scope_report.py deleted file mode 100644 index 978ac7a1d5..0000000000 --- a/src/sqlbuild/compiler/scopes/main/serialize_scope_report.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Public canonical scope report serialization operation.""" - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ScopeBrowseResult, ScopeListResult, ScopeReport - - -def serialize_scope_report(*, report: ScopeReport | ScopeBrowseResult | ScopeListResult) -> str: - """Return deterministic schema-version-one JSON for any scope query result.""" - - return _native.serialize_scope_report(report) diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py index 843d59146f..7af6ef3860 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py +++ b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py @@ -9,11 +9,10 @@ import orjson import sqlbuild._native as _native -from sqlbuild._native import compiled_project_facts, declaration_rows +from sqlbuild._native import compiled_project_facts, declaration_rows, scope_metadata_projection from sqlbuild.adapter.type_system.main.types_equal import types_equal from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity -from sqlbuild.compiler.scopes.main.scope_metadata import scope_metadata_projection from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine.constants import ( NATIVE_RULES_CACHE_FILE, diff --git a/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py b/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py index 5cfa7d863b..ca7ba5884e 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py @@ -34,14 +34,6 @@ class ScopeCommandParityTestCase: expected_minimum_grants: int -@dataclass(frozen=True) -class ScopeEngineTierTestCase: - """How often building the declaration scope opens the native index.""" - - description: str - expected_native_attempts: int - - @dataclass(frozen=True) class ExpectedNameScanTestCase: """Seeded SQL test bodies whose relationship scans match Python's recorded golden.""" diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index 222e8961d4..0a0558e330 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -17,6 +17,8 @@ build_declaration_scope, discover_project_inputs, resolve_declaration_context, + resolve_scope_declaration_visibility, + resolve_scope_path_visibility, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros @@ -33,15 +35,7 @@ DiscoveredProjectInputs, EnumDeclaration, ) -from sqlbuild.compiler.scopes.main import _open_native_scope_index as open_index_module -from sqlbuild.compiler.scopes.main._declaration_lexical_path import declaration_lexical_path -from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index -from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( - resolve_scope_declaration_visibility, -) -from sqlbuild.compiler.scopes.main._resolve_scope_path_visibility import ( - resolve_scope_path_visibility, -) +from sqlbuild.compiler.scopes.constants import CURRENT_PATH_COMPONENT, DECLARATION_LEXICAL_STEM from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, @@ -312,32 +306,6 @@ def scope_command_index(*, project_dir: Path) -> ScopeIndex: return _native.load_or_build_scope_index(project_dir=project_dir, no_cache=True) -def native_scope_attempts( - *, project_dir: Path, monkeypatch: pytest.MonkeyPatch -) -> tuple[int, ScopeOutcome]: - """Build the declaration scope; count how often the native index was opened.""" - - discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) - macros: dict[str, LoadedMacro] = load_project_macros(discovered.macro_files) - attempts: list[None] = [] - - def counted( - *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: Mapping[str, LoadedMacro] - ) -> _native.NativeScopeIndex: - attempts.append(None) - return open_native_scope_index( - discovered_inputs=discovered_inputs, loaded_macros=loaded_macros - ) - - monkeypatch.setattr(open_index_module, "open_native_scope_index", counted) - outcome: ScopeOutcome = _scope_outcome( - discovered=discovered, - macros=macros, - syntax=DuckDbAdapter().sql_lexical_syntax, - ) - return len(attempts), outcome - - def _scope_outcome( *, discovered: DiscoveredProjectInputs, @@ -769,7 +737,9 @@ def _path_visibility( _ = resolution lexical_path: Path = next( map( - lambda record: Path(declaration_lexical_path(record=record)), + lambda record: ( + Path(record.owning_path or CURRENT_PATH_COMPONENT) / DECLARATION_LEXICAL_STEM + ), filter( lambda record: ( record.path == target_path.as_posix() and record.scope is not ScopeKind.PRIVATE diff --git a/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py b/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py index c427e57c6c..c0bdd44c72 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py @@ -12,13 +12,11 @@ GeneratedScopeParityTestCase, ScopeCommandParityTestCase, ScopeEngineParityTestCase, - ScopeEngineTierTestCase, ) from tests.integration.src.sqlbuild.compiler.scopes.helpers import ( SCOPED_PROJECT, ScopeOutcome, generated_scope_files, - native_scope_attempts, scope_command_index, scope_outcome, write_project, @@ -198,24 +196,5 @@ def test_given_scoped_project_when_scope_command_builds_index_then_grants_are_ke assert len(index.grants) >= test_case.expected_minimum_grants, test_case.description -@pytest.mark.parametrize( - "test_case", - [ - ScopeEngineTierTestCase( - description="native declaration scopes", expected_native_attempts=1 - ), - ], - ids=lambda case: case.description, -) -def test_given_scoped_project_when_building_scope_then_the_native_index_is_built_once( - test_case: ScopeEngineTierTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - write_project(project_dir=tmp_path, files=SCOPED_PROJECT) - - attempts, outcome = native_scope_attempts(project_dir=tmp_path, monkeypatch=monkeypatch) - - assert (attempts, outcome.kind) == (test_case.expected_native_attempts, "ok") - - if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py index 7f8d8e2281..fbc50afd1e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py @@ -4,9 +4,7 @@ from dataclasses import replace -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup -from sqlbuild.compiler.scopes.main.preview_scope_move import preview_scope_move -from sqlbuild.compiler.scopes.main.query_scope_report import query_scope_report +from sqlbuild._native import build_scope_lookup, preview_scope_move, query_scope_report from sqlbuild.compiler.scopes.models import ( OwnershipRoot, ResourceIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index d01b4e432b..44dade0b57 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -9,6 +9,7 @@ assemble_project, build_compile_inputs, build_native_declaration_contexts, + build_scope_lookup, discover_project_inputs, expand_sql_macros, infer_native_query_columns, @@ -63,7 +64,6 @@ from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.planner._helpers.graph.core import build_execution_upstream_deps from sqlbuild.compiler.planner._helpers.resolve.refs import resolve_ref_references -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py index 9d6cdffbaf..541ed2451e 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py @@ -9,7 +9,12 @@ import pytest -from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs +from sqlbuild._native import ( + assemble_project, + build_compile_inputs, + discover_project_inputs, + scope_metadata_projection, +) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView, ModelInput from sqlbuild.compiler.discovery.models import ( @@ -17,7 +22,6 @@ DiscoveredSourceFile, SqlHookEntry, ) -from sqlbuild.compiler.scopes.main.scope_metadata import scope_metadata_projection from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, ResourceIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py index 315de7de6a..96ae930421 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py @@ -8,13 +8,13 @@ from itertools import compress from pathlib import PurePath -from sqlbuild.compiler.scopes._helpers.paths import normalize_path -from sqlbuild.compiler.scopes.main._query_scope_target import query_scope_target -from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( +from sqlbuild._native import ( + build_scope_lookup, + normalize_scope_path, + query_scope_target, resolve_scope_declaration_visibility, + resolve_scope_visibility, ) -from sqlbuild.compiler.scopes.main._resolve_scope_visibility import resolve_scope_visibility -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, @@ -156,7 +156,7 @@ def _targets(lookup: ScopeLookup) -> list[ResourceIdentity | str]: def _parent(path: str) -> str: - return posixpath.dirname(normalize_path(path=path)) or "." + return posixpath.dirname(normalize_scope_path(path=path)) or "." def _is_descendant(*, path: str, ancestor: str) -> bool: @@ -166,7 +166,7 @@ def _is_descendant(*, path: str, ancestor: str) -> bool: def _reference_reason( *, resource: ResourceRecord, declaration: DeclarationRecord ) -> VisibilityReason | None: - owner: str = normalize_path(path=declaration.owning_path or ".") + owner: str = normalize_scope_path(path=declaration.owning_path or ".") parent: str = _parent(resource.path) visible: dict[ScopeKind, bool] = { ScopeKind.GLOBAL: True, @@ -180,7 +180,7 @@ def _reference_reason( def _reference_inaccessible_reason( *, resource: ResourceRecord, declaration: DeclarationRecord ) -> InaccessibleReason: - owner: str = normalize_path(path=declaration.owning_path or ".") + owner: str = normalize_scope_path(path=declaration.owning_path or ".") candidates: tuple[tuple[bool, InaccessibleReason], ...] = ( (declaration.scope is ScopeKind.PRIVATE, InaccessibleReason.PRIVATE_OWNER), (declaration.scope is ScopeKind.LOCAL, InaccessibleReason.LOCAL_BOUNDARY), diff --git a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py index e420ec69c1..36c6d58db8 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py @@ -7,15 +7,15 @@ from pathlib import Path from typing import cast +from sqlbuild._native import ( + build_scope_lookup, + get_placement_validated_scope_index, +) from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, DiscoveredSqlModelFile, EnumDeclaration, ) -from sqlbuild.compiler.scopes.main._get_placement_validated_scope_index import ( - get_placement_validated_scope_index, -) -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup from sqlbuild.compiler.scopes.models import ( ConstantMetadata, DeclarationIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py index 2e7c130848..61e6cc51c3 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py @@ -6,6 +6,13 @@ import pytest +from sqlbuild._native import ( + build_scope_index, + build_scope_lookup, + query_scope_target, + resolve_scope_visibility, + validate_scope_index, +) from sqlbuild.compiler.compile.models import LoadedMacro from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, @@ -24,11 +31,6 @@ EnumMember, ) from sqlbuild.compiler.scopes.exceptions import ScopeValidationError -from sqlbuild.compiler.scopes.main._build_scope_index import build_scope_index -from sqlbuild.compiler.scopes.main._query_scope_target import query_scope_target -from sqlbuild.compiler.scopes.main._resolve_scope_visibility import resolve_scope_visibility -from sqlbuild.compiler.scopes.main._validate_scope_index import validate_scope_index -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py b/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py index e8263cb7de..49ea1b4695 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py @@ -4,10 +4,8 @@ import pytest +from sqlbuild._native import build_scope_lookup, explain_scope_declaration, query_scope_report from sqlbuild.compiler.scopes.exceptions import InvalidQualifiedIdentityError -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup -from sqlbuild.compiler.scopes.main.explain_scope_declaration import explain_scope_declaration -from sqlbuild.compiler.scopes.main.query_scope_report import query_scope_report from sqlbuild.compiler.scopes.models import ( DeclarationExplanation, DeclarationIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py b/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py index 762a66aa14..700a8bf189 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py @@ -8,8 +8,7 @@ import pytest -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup -from sqlbuild.compiler.scopes.main.scope_metadata import scope_metadata_projection +from sqlbuild._native import build_scope_lookup, scope_metadata_projection from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py b/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py index 7abb2d4126..2542649660 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.scopes._helpers.paths import normalize_path +from sqlbuild._native import normalize_scope_path from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError from tests.unit.src.sqlbuild.compiler.scopes._test_types import PathNormalizationCase @@ -33,7 +33,7 @@ def test_given_posix_or_windows_path_when_normalizing_then_returns_canonical_posix_path( test_case: PathNormalizationCase, ) -> None: - assert normalize_path(path=test_case.path) == test_case.expected_path + assert normalize_scope_path(path=test_case.path) == test_case.expected_path @pytest.mark.parametrize( @@ -49,7 +49,7 @@ def test_given_path_outside_project_when_normalizing_then_rejects_it( test_case: PathNormalizationCase, ) -> None: with pytest.raises(InvalidScopePathError, match=test_case.expected_path): - normalize_path(path=test_case.path) + normalize_scope_path(path=test_case.path) if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py b/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py index 03fdcdb665..532f665da8 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py @@ -4,11 +4,11 @@ import pytest -from sqlbuild.compiler.scopes.exceptions import ScopeValidationError -from sqlbuild.compiler.scopes.main._get_placement_validated_scope_index import ( +from sqlbuild._native import ( get_placement_validated_scope_index, + validate_scope_index, ) -from sqlbuild.compiler.scopes.main._validate_scope_index import validate_scope_index +from sqlbuild.compiler.scopes.exceptions import ScopeValidationError from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py b/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py index 8f76f0ce39..0a508e3933 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.scopes.main.relocate_declarations_for_move import ( +from sqlbuild._native import ( relocate_declarations_for_move, ) from sqlbuild.compiler.scopes.models import ( diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_reports.py b/tests/unit/src/sqlbuild/compiler/scopes/test_reports.py index f540473037..79ea620d89 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_reports.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_reports.py @@ -9,13 +9,15 @@ import pytest -from sqlbuild.compiler.scopes.main.browse_scope_folders import browse_scope_folders -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup -from sqlbuild.compiler.scopes.main.explain_scope_declaration import explain_scope_declaration -from sqlbuild.compiler.scopes.main.list_scope_declarations import list_scope_declarations -from sqlbuild.compiler.scopes.main.preview_scope_move import preview_scope_move -from sqlbuild.compiler.scopes.main.query_scope_report import query_scope_report -from sqlbuild.compiler.scopes.main.serialize_scope_report import serialize_scope_report +from sqlbuild._native import ( + browse_scope_folders, + build_scope_lookup, + explain_scope_declaration, + list_scope_declarations, + preview_scope_move, + query_scope_report, + serialize_scope_report, +) from sqlbuild.compiler.scopes.models import ( OwnershipRoot, ResourceRecord, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_scope_facts.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_scope_facts.py index 8c7b878fdf..c35037745b 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_scope_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_scope_facts.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild.compiler.scopes.main.scope_metadata import scope_metadata_projection +from sqlbuild._native import scope_metadata_projection from sqlbuild.compiler.scopes.models import DeclarationRecord, ScopeCompleteness, ScopeIndex from tests.unit.src.sqlbuild.compiler.scopes.helpers import scope_index from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ScopePayloadTestCase From c0843fc77192394041b05f2e9c499229202a0d42 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 05:36:46 +0100 Subject: [PATCH 29/40] refactor: move the compile entry glue into native bindings --- .../_helpers/analysis/required_columns.rs | 60 ++++-- .../_helpers/build_command/invocation.rs | 2 +- .../_helpers/command_runtime/connection.rs | 2 +- .../_helpers/command_runtime/warehouses.rs | 2 +- .../compile_attachment/effective_config.rs | 101 ++++++++- .../_helpers/compile_attachment/mod.rs | 1 + .../compile_attachment/registration.rs | 23 ++- .../sql_expansion_context.rs | 195 ++++++++++++++++++ .../_helpers/compile_command/report.rs | 7 +- .../compile_driver/assembly_project.rs | 7 +- .../_helpers/compile_driver/pipeline.rs | 2 +- .../_helpers/compile_driver/targets.rs | 4 +- .../bindings/_helpers/cost_command/refresh.rs | 2 +- .../bindings/_helpers/diff_report/query.rs | 4 +- .../bindings/_helpers/load/load_command.rs | 6 +- .../bindings/_helpers/load/load_selection.rs | 2 +- .../_helpers/scenario_command/test_command.rs | 2 +- .../_helpers/scopes/compile_usages.rs | 2 +- .../bindings/_helpers/scopes/scope_loading.rs | 10 +- .../_helpers/scopes/scope_validation.rs | 55 ++++- .../_helpers/semantic_shapes/coverage.rs | 2 +- scripts/fensu_policy/constants.py | 1 - src/sqlbuild/_native.pyi | 34 ++- .../cli/commands/_helpers/plan/invocation.py | 7 +- .../compile/_helpers/render/user_macros.py | 2 +- .../main/_analyze_columns_and_lineage.py | 29 --- .../compile/main/_effective_target.py | 28 --- .../main/_effective_target_namespace.py | 28 --- .../main/_scope_index_with_compile_usages.py | 39 ---- .../compile/main/_semantic_coverage.py | 9 - .../compiler/compile/main/_source_bindings.py | 19 -- .../compile/main/_static_macro_inventory.py | 15 -- .../compiler/compile/main/effective_config.py | 46 ----- .../compile/main/effective_runtime.py | 41 ---- .../compile/main/effective_settings.py | 20 -- .../compile/main/sql_expansion_context.py | 120 ----------- .../planner/_helpers/fixtures/completion.py | 4 +- .../planner/_helpers/output/plan_entry.py | 4 +- .../dbt/_helpers/pipeline/execution_phases.py | 2 +- .../dbt/_helpers/pipeline/plan_output.py | 2 +- src/sqlbuild/lint/_helpers/expansion.py | 2 +- .../sqlbuild/cli/commands/main/test_rules.py | 8 +- .../pipeline/test_native_binding_catalog.py | 13 +- .../compiler/pipeline/test_semantic_triage.py | 9 +- .../rules/test_canonical_operations.py | 2 +- .../commands/_helpers/cost/test_refresh.py | 2 +- .../compile/_helpers/test_required_columns.py | 4 +- .../compile/test_effective_runtime.py | 3 +- .../lint/test_adapter_lexical_syntax.py | 8 +- 49 files changed, 503 insertions(+), 489 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs delete mode 100644 src/sqlbuild/compiler/compile/main/_analyze_columns_and_lineage.py delete mode 100644 src/sqlbuild/compiler/compile/main/_effective_target.py delete mode 100644 src/sqlbuild/compiler/compile/main/_effective_target_namespace.py delete mode 100644 src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py delete mode 100644 src/sqlbuild/compiler/compile/main/_semantic_coverage.py delete mode 100644 src/sqlbuild/compiler/compile/main/_source_bindings.py delete mode 100644 src/sqlbuild/compiler/compile/main/_static_macro_inventory.py delete mode 100644 src/sqlbuild/compiler/compile/main/effective_config.py delete mode 100644 src/sqlbuild/compiler/compile/main/effective_runtime.py delete mode 100644 src/sqlbuild/compiler/compile/main/effective_settings.py delete mode 100644 src/sqlbuild/compiler/compile/main/sql_expansion_context.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs index 0deadf4ef8..c7620efbe8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs @@ -1,30 +1,38 @@ //! The external columns a model's query reads. use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::required_columns::errors::RequiredColumnsError; use sqlbuild_analysis::required_columns::main::resolve_required_columns::resolve_required_columns as resolve; use sqlbuild_analysis::required_columns::models::{ColumnRead, ReferenceInput}; use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::types::CompilerDetach; -/// Each `(resource type, resource, column)` the query reads, given its `(kind, name)` references. +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; + +/// CTE-aware column reads of `query_sql`, resolved to the external resources it references. #[pyfunction] -#[pyo3(signature = (query_sql, references, dialect))] -fn resolve_required_columns( - py: Python<'_>, +#[pyo3(signature = (*, query_sql, references, dialect))] +fn analyze_resolved_column_reads<'py>( + py: Python<'py>, query_sql: &str, - references: Vec<(String, String)>, + references: &Bound<'py, PyAny>, dialect: Option<&str>, -) -> PyResult> { - let references: Vec = references - .into_iter() - .map(|(kind, name)| ReferenceInput { kind, name }) - .collect(); +) -> PyResult> { + let mut inputs: Vec = Vec::new(); + for reference in references.try_iter()? { + let reference: Bound<'py, PyAny> = reference?; + inputs.push(ReferenceInput { + kind: reference.getattr("ref_kind")?.str()?.extract()?, + name: reference.getattr("ref_name")?.extract()?, + }); + } let reads: Vec = py - .compiler_detach(|| Ok(resolve(query_sql, &references, dialect))) + .compiler_detach(|| Ok(resolve(query_sql, &inputs, dialect))) .map_err(value_error)? .map_err(|error| match error { RequiredColumnsError::UnknownDialect(name) => { @@ -32,12 +40,30 @@ fn resolve_required_columns( } RequiredColumnsError::Normalization(message) => value_error(message), })?; - Ok(reads - .into_iter() - .map(|read| (read.resource_type, read.resource_name, read.column_name)) - .collect()) + let fact: Bound<'py, PyAny> = imported(py, COMPILE_MODELS_MODULE, "CompiledLineageSourceFact")?; + let mut facts: Vec> = Vec::with_capacity(reads.len()); + for read in reads { + facts.push(call_with( + &fact, + &[ + ( + "resource_type", + PyString::new(py, &read.resource_type).into_any(), + ), + ( + "resource_name", + PyString::new(py, &read.resource_name).into_any(), + ), + ( + "column_name", + PyString::new(py, &read.column_name).into_any(), + ), + ], + )?); + } + PyTuple::new(py, facts) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_required_columns, module)?) + module.add_function(wrap_pyfunction!(analyze_resolved_column_reads, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 94492d1078..3d109f203a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -15,7 +15,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const REPORTERS_MODULE: &str = "sqlbuild.cli.progress.main._build_command_progress_reporters"; -const SETTINGS_MODULE: &str = "sqlbuild.compiler.compile.main.effective_settings"; +const SETTINGS_MODULE: &str = "sqlbuild._native"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 2b888b8f70..0ab1a951c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -20,7 +20,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const DBT_PROFILE_MODULE: &str = "sqlbuild.integrations.dbt.main.profile.profile_connection"; const DBT_ERRORS_MODULE: &str = "sqlbuild.integrations.dbt.exceptions"; -const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; +const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index c61a7a7923..e02035283c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -32,7 +32,7 @@ const SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_sco const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main._session_warehouse_support"; -const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; +const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild._native"; const RESOLVE_OPTIONS: (&[&str], &[&str]) = ( &[ "discovered_inputs", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs index a213be3c23..96bf6589cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs @@ -2,13 +2,16 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyfunction; -use pyo3::types::{PyDict, PyList}; +use pyo3::types::{PyDict, PyList, PyTuple}; use sqlbuild_model_config::templates::models::TemplateOptions; use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::contracts::spec_resolution::{ + resolve_target_config, resolve_target_name, +}; use crate::bindings::_helpers::model_config::template_data::{ - expanded_effective_vars, expanded_template_data, + expanded_effective_vars, expanded_template_data, run_id, }; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -91,6 +94,100 @@ pub(crate) fn build_effective_vars<'py>( expanded_effective_vars(merged.downcast()?) } +/// The selected target name of a discovered project and its effective config, if any. +fn target_selection<'py>( + discovered: &Bound<'py, PyAny>, + selected: Option>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = discovered.py(); + let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; + let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; + let selected: Bound<'py, PyAny> = selected.unwrap_or_else(|| py.None().into_bound(py)); + let Some(name) = resolve_target_name(py, &project_config, &local_config, &selected)? else { + return Ok((py.None().into_bound(py), py.None().into_bound(py))); + }; + let name: Bound<'py, PyAny> = name.into_any(); + let config: Bound<'py, PyAny> = + resolve_target_config(py, &project_config, &local_config, &name)?; + Ok((name, config)) +} + +/// The effective vars of a discovered project for `target_config` and optional CLI vars. +fn discovered_vars<'py>( + discovered: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, + cli_vars: Option>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let cli_vars: Bound<'py, PyAny> = match cli_vars { + Some(cli_vars) if !cli_vars.is_none() => cli_vars, + _ => PyDict::new(py).into_any(), + }; + build_effective_vars( + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, + target_config, + &cli_vars, + ) +} + +/// Build effective target config without compiling resources. +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, selected_target = None))] +pub(crate) fn build_effective_target_config<'py>( + discovered_inputs: &Bound<'py, PyAny>, + selected_target: Option>, +) -> PyResult> { + Ok(target_selection(discovered_inputs, selected_target)?.1) +} + +/// Build the effective project connection config without compiling resources. +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, selected_target = None, cli_vars = None))] +pub(crate) fn build_effective_connection_config<'py>( + discovered_inputs: &Bound<'py, PyAny>, + selected_target: Option>, + cli_vars: Option>, +) -> PyResult> { + let (_, target_config) = target_selection(discovered_inputs, selected_target)?; + let effective_vars: Bound<'py, PyAny> = + discovered_vars(discovered_inputs, &target_config, cli_vars)?; + build_effective_connection( + &discovered_inputs.getattr("project_config")?, + &discovered_inputs.getattr("local_config")?, + &target_config, + &effective_vars, + ) +} + +/// Build environment, vars and invocation id without compiling resources. +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, selected_target = None, cli_vars = None))] +pub(crate) fn build_effective_runtime_config<'py>( + discovered_inputs: &Bound<'py, PyAny>, + selected_target: Option>, + cli_vars: Option>, +) -> PyResult> { + let py: Python<'py> = discovered_inputs.py(); + let (target_name, target_config) = target_selection(discovered_inputs, selected_target)?; + let effective_vars: Bound<'py, PyAny> = + discovered_vars(discovered_inputs, &target_config, cli_vars)?; + let run_id: Bound<'py, PyAny> = run_id(&py.None().into_bound(py))?; + PyTuple::new(py, [target_name, effective_vars, run_id]) +} + +/// Build effective settings without compiling resources. +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs))] +pub(crate) fn build_effective_settings_config<'py>( + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + build_effective_settings( + &discovered_inputs.getattr("project_config")?, + &discovered_inputs.getattr("local_config")?, + ) +} + /// `build_effective_vars` for a discovered project and target, without CLI vars. pub(crate) fn project_vars<'py>( discovered: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs index 9225615d45..f9cea53494 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs @@ -4,5 +4,6 @@ pub(crate) mod effective_config; pub(crate) mod model_config; pub(crate) mod model_inputs; pub(crate) mod registration; +mod sql_expansion_context; mod storage_policies; pub(crate) mod target; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs index a5a741456a..ea6f752684 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs @@ -4,10 +4,15 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; use crate::bindings::_helpers::compile_attachment::{ - cursor_roles, declaration_scope, effective_config, model_config, model_inputs, target, + cursor_roles, declaration_scope, effective_config, model_config, model_inputs, + sql_expansion_context, target, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + sql_expansion_context::build_sql_expansion_context, + module + )?)?; module.add_function(wrap_pyfunction!( effective_config::build_effective_connection, module @@ -20,6 +25,22 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { effective_config::build_effective_vars, module )?)?; + module.add_function(wrap_pyfunction!( + effective_config::build_effective_target_config, + module + )?)?; + module.add_function(wrap_pyfunction!( + effective_config::build_effective_connection_config, + module + )?)?; + module.add_function(wrap_pyfunction!( + effective_config::build_effective_runtime_config, + module + )?)?; + module.add_function(wrap_pyfunction!( + effective_config::build_effective_settings_config, + module + )?)?; module.add_function(wrap_pyfunction!( model_config::native_model_config_session, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs new file mode 100644 index 0000000000..5bed96ebad --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs @@ -0,0 +1,195 @@ +//! The SQL expansion inputs lint uses to expand authored SQL outside a compile. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const USER_MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; + +/// Build the declaration scope of `discovered`; a static scope lends only its index to macros. +fn declaration_scope<'py>( + discovered: &Bound<'py, PyAny>, + static_scope: Bound<'py, PyAny>, + sql_lexical_syntax: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let loaded_macros: Bound<'py, PyAny> = imported(py, USER_MACROS_MODULE, "load_project_macros")? + .call1((discovered.getattr("macro_files")?,))?; + if !static_scope.is_none() { + let rebound: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "rebind_declaration_scope")?, + &[ + ("scope", static_scope), + ("discovered_inputs", discovered.clone()), + ("loaded_macros", loaded_macros.clone()), + ], + )?; + if !rebound.is_none() { + return Ok(rebound); + } + } + call_with( + &imported(py, NATIVE_MODULE, "build_declaration_scope")?, + &[ + ("discovered_inputs", discovered.clone()), + ("loaded_macros", loaded_macros), + ("sql_lexical_syntax", sql_lexical_syntax), + ], + ) +} + +/// Each model file's own enum and constant declarations, by file path. +fn local_declarations<'py>(discovered: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = discovered.py(); + let context: Bound<'py, PyAny> = + imported(py, COMPILE_MODELS_MODULE, "DeclarationResolutionContext")?; + let by_name = |declarations: Bound<'py, PyAny>| -> PyResult> { + let named: Bound<'py, PyDict> = PyDict::new(py); + for declaration in declarations.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + named.set_item(declaration.getattr("name")?, declaration)?; + } + Ok(named.into_any()) + }; + let local: Bound<'py, PyDict> = PyDict::new(py); + for model_file in discovered.getattr("model_files")?.try_iter()? { + let model_file: Bound<'py, PyAny> = model_file?; + local.set_item( + model_file.getattr("file_path")?, + call_with( + &context, + &[ + ("enums", by_name(model_file.getattr("enum_declarations")?)?), + ( + "constants", + by_name(model_file.getattr("constant_declarations")?)?, + ), + ], + )?, + )?; + } + Ok(local) +} + +/// Assemble SQL expansion inputs; a static scope lends only its index to private macros. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_sql_expansion_context<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_sql_expansion_context", + options, + ( + &["project_dir", "value_renderer"], + &[ + "cli_vars", + "discovered_inputs", + "declaration_scope", + "static_declaration_scope", + ], + ), + )?; + let value_renderer: Bound<'py, PyAny> = arguments.value("value_renderer")?; + let mut discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; + if discovered.is_none() { + discovered = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[ + ("project_dir", arguments.value("project_dir")?), + ( + "sql_analysis_enabled_override", + PyBool::new(py, false).to_owned().into_any(), + ), + ], + )?; + } + let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; + let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; + let mut cli_vars: Bound<'py, PyAny> = arguments.value("cli_vars")?; + if cli_vars.is_none() { + cli_vars = PyDict::new(py).into_any(); + } + let effective_vars: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "build_effective_vars")?, + &[ + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ("target_config", py.None().into_bound(py)), + ("cli_vars", cli_vars), + ], + )?; + let mut scope: Bound<'py, PyAny> = arguments.value("declaration_scope")?; + if scope.is_none() { + scope = declaration_scope( + &discovered, + arguments.value("static_declaration_scope")?, + value_renderer.getattr("sql_lexical_syntax")?, + )?; + } + let (enums, constants): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + imported(py, NATIVE_MODULE, "public_declaration_indexes")? + .call1((&discovered,))? + .extract()?; + let collection_rendering: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_effective_collection_rendering")?, + &[ + ("project_config", project_config.clone()), + ("declaration_override", py.None().into_bound(py)), + ], + )?; + let macro_context: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "MacroContext")?, + &[ + ( + "adapter_name", + call_with( + &imported(py, NATIVE_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", project_config.clone()), + ("local_config", local_config), + ], + )?, + ), + ( + "sql_analysis_enabled", + PyBool::new(py, false).to_owned().into_any(), + ), + ("target_name", project_config.getattr("default_target")?), + ("vars", effective_vars.clone()), + ("_value_renderer", value_renderer.clone()), + ("_collection_rendering", collection_rendering.clone()), + ( + "_enforce_explicit_references", + project_config + .getattr("references")? + .getattr("enforce_explicit")?, + ), + ], + )?; + call_with( + &imported(py, COMPILE_MODELS_MODULE, "SqlExpansionContext")?, + &[ + ("effective_vars", effective_vars), + ("loaded_macros", scope.getattr("loaded_macros")?), + ("macro_context", macro_context), + ("enums", enums), + ("constants", constants), + ("value_renderer", value_renderer), + ("collection_rendering", collection_rendering), + ( + "local_declarations", + local_declarations(&discovered)?.into_any(), + ), + ("declaration_resolver", scope.getattr("resolver")?), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index c7b0e058e3..18eeddf21a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -19,8 +19,8 @@ use crate::bindings::_helpers::compile_outputs::json_report::{ emit_json_report, emit_orjson_report, }; use crate::bindings::_helpers::discovery::hook_entries::hook_entries; +use crate::bindings::_helpers::semantic_shapes::coverage::semantic_coverage; -const SEMANTIC_COVERAGE_MODULE: &str = "sqlbuild.compiler.compile.main._semantic_coverage"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; @@ -163,10 +163,7 @@ impl<'py> ReportInputs<'py> { if !self.sql_validation_enabled { return Ok(PyDict::new(py)); } - let reasons: Bound<'py, PyDict> = - imported(py, SEMANTIC_COVERAGE_MODULE, "get_semantic_coverage")? - .call1((&self.project,))? - .downcast_into()?; + let reasons: Bound<'py, PyDict> = semantic_coverage(&self.project)?; let Some(names) = self.selected_names(MODEL_TYPE)? else { return Ok(reasons); }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index ec227ec603..16a0ce90e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -18,9 +18,9 @@ use crate::bindings::_helpers::compile_driver::assembly_models::{ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::model_config::template_data::early_model_templates; use crate::bindings::_helpers::scopes::scope_errors::report_scope_index_errors; +use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; const NATIVE_MODULE: &str = "sqlbuild._native"; -const SCOPE_USAGES_MODULE: &str = "sqlbuild.compiler.compile.main._scope_index_with_compile_usages"; pub(crate) const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTER_TYPES_MODULE: &str = "sqlbuild.adapter.contract.types"; const SCENARIO_CONFIG_MODULE: &str = "sqlbuild._native"; @@ -103,10 +103,7 @@ pub(crate) fn assembled_project<'py>( ], )?; let resources: Bound<'py, PyAny> = native.getattr("resources")?; - let scope_index: Bound<'py, PyAny> = call_with( - &imported(py, SCOPE_USAGES_MODULE, "scope_index_with_compile_usages")?, - &[("inputs", inputs.clone())], - )?; + let scope_index: Bound<'py, PyAny> = scope_index_with_compile_usages(inputs)?; report_scope_index_errors(&scope_index)?; let target_values: Bound<'py, PyAny> = effective_target_values(inputs)?; let models: Bound<'py, PyTuple> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index 70bee37f71..fbe1f63570 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::frontier::stage_capture::write_plan_capture; const NATIVE_MODULE: &str = "sqlbuild._native"; -const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; +const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild._native"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index ab1b1411f9..ab172b2417 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -28,8 +28,8 @@ use crate::bindings::_helpers::model_config::template_data::expanded_template_da const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; -const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; -const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; +const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild._native"; +const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild._native"; const SETTING_HELP_MODULE: &str = "sqlbuild.errors.setting_help.main.setting_help"; /// Connection settings that identify the warehouse two targets write to. const CONNECTION_IDENTITY_KEYS: [&str; 7] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs index c7ae5a8a37..073bca79ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs @@ -12,7 +12,7 @@ use sqlbuild_core::cost_command::models::CostCoverage; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild.compiler.compile.main._effective_target"; +const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const COST_TYPES: &str = "sqlbuild.cost.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index 7a9c977cd8..ad318afc21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -20,8 +20,8 @@ use crate::bindings::_helpers::diff_report::support::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; -const NAMESPACE_MODULE: &str = "sqlbuild.compiler.compile.main._effective_target_namespace"; -const RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; +const NAMESPACE_MODULE: &str = "sqlbuild._native"; +const RUNTIME_MODULE: &str = "sqlbuild._native"; const POLYGLOT_MODULE: &str = "sqlbuild.compiler.sql_analysis.main.import_polyglot_sql"; const COMMAND_MODELS_MODULE: &str = "sqlbuild.cli.commands.models"; const LIFECYCLE_MODULE: &str = "sqlbuild.executor.diff.classes.query_artifact_lifecycle"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 97791c63fb..54353b4c43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -28,9 +28,9 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild.compiler.compile.main._effective_target"; -const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; -const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild.compiler.compile.main.effective_settings"; +const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild._native"; +const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild._native"; +const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index af5cb07dd4..7b66a5d255 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; const DESTINATION_MODULE: &str = "sqlbuild._native"; -const CONNECTION_CONFIG_MODULE: &str = "sqlbuild.compiler.compile.main.effective_config"; +const CONNECTION_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTER_TYPES_MODULE: &str = "sqlbuild.adapter.contract.types"; const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index 004178139b..05eefdb218 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -27,7 +27,7 @@ use crate::bindings::_helpers::scenario_command::setup::{ }; use crate::bindings::_helpers::scenario_command::snapshot_sync::{LocalSync, sync_local_snapshots}; -const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild.compiler.compile.main.effective_settings"; +const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild._native"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const SCENARIO_MODELS: &str = "sqlbuild.executor.scenario.models"; const LOCAL_ADAPTER: &str = "duckdb"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs index 33705c8763..ec680f1ab9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -87,7 +87,7 @@ fn diagnostic_row(diagnostic: InaccessibleDiagnostic) -> DiagnosticRow { /// `model_schema_scope_usages`: each schema's parent and enum `UsageRecord`s; S006s are reported. #[pyfunction] -fn model_schema_scope_usages<'py>( +pub(crate) fn model_schema_scope_usages<'py>( schema_files: &Bound<'py, PyAny>, resolver: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index ec270ed060..37bee60bec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -16,6 +16,7 @@ use crate::bindings::_helpers::scopes::scope_cache::{ }; use crate::bindings::_helpers::scopes::scope_index::open_native_scope_index; use crate::bindings::_helpers::scopes::scope_tolerant::scope_tolerant_index; +use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; const NATIVE_MODULE: &str = "sqlbuild._native"; const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; @@ -27,9 +28,7 @@ const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.adapter.discovery.main.resolve_ad const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; -const COMPILE_USAGES_MODULE: &str = - "sqlbuild.compiler.compile.main._scope_index_with_compile_usages"; -const MACRO_INVENTORY_MODULE: &str = "sqlbuild.compiler.compile.main._static_macro_inventory"; +const MACRO_INVENTORY_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; /// Where the scope index cache file of a project lives: `(directory, file name)`. fn cache_location(project_dir: &Bound<'_, PyAny>) -> PyResult<(PathBuf, String)> { @@ -133,10 +132,7 @@ fn compiled_index<'py>( ("python_stage", python_stage), ], )?; - call_with( - &imported(py, COMPILE_USAGES_MODULE, "scope_index_with_compile_usages")?, - &[("inputs", compile_inputs)], - ) + scope_index_with_compile_usages(&compile_inputs) } /// The partial static index of tolerant discovery, with its faults as diagnostics. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs index fc76ea4a16..5d969e577f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs @@ -15,7 +15,7 @@ use sqlbuild_scopes::scope_index::models::ResourceIdentity; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::scopes::compile_usages::{ - MergeRows, SourceRow, UsageRow, merged_usage_rows, + MergeRows, SourceRow, UsageRow, merged_usage_rows, model_schema_scope_usages, }; use crate::bindings::_helpers::scopes::placement::{QualifiedRow, resource_identity}; use crate::bindings::_helpers::scopes::scope_objects::{ @@ -28,6 +28,14 @@ const DATACLASSES_MODULE: &str = "dataclasses"; const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; const RUNTIME_USAGE: usize = 2; +/// Compile input groups whose items carry declaration usages, after the model inputs. +const COMPILE_USAGE_GROUPS: [&str; 5] = [ + "source_inputs", + "sql_function_inputs", + "audit_inputs", + "test_inputs", + "scenario_inputs", +]; /// `dataclasses.replace(item, **changes)`. pub(crate) fn replace<'py>( @@ -319,6 +327,51 @@ pub(crate) fn scope_relocated_declarations( Ok(Some(PyTuple::new(py, moved)?.unbind())) } +fn usage_records<'py>(records: &Bound<'py, PyAny>) -> PyResult>> { + records.try_iter()?.collect() +} + +/// Compile inputs' canonical index with all usage and placement facts. +pub(crate) fn scope_index_with_compile_usages<'py>( + inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let scope: Bound<'py, PyAny> = inputs.getattr("declaration_scope")?; + let resolver: Bound<'py, PyAny> = if scope.is_none() { + scope + } else { + scope.getattr("resolver")? + }; + let schema_files: Bound<'py, PyAny> = inputs + .getattr("discovered_inputs")? + .getattr("model_schema_files")?; + let mut usages: Vec> = Vec::new(); + usages.extend(usage_records( + model_schema_scope_usages(&schema_files, &resolver)?.as_any(), + )?); + for model in inputs.getattr("model_inputs")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + usages.extend(usage_records(&model.getattr("macro_usages")?)?); + usages.extend(usage_records(&model.getattr("declaration_usages")?)?); + } + for group in COMPILE_USAGE_GROUPS { + for item in inputs.getattr(group)?.try_iter()? { + usages.extend(usage_records(&item?.getattr("declaration_usages")?)?); + } + } + let enforce_placement: bool = inputs + .getattr("project_config")? + .getattr("scopes")? + .getattr("enforce_placement")? + .extract()?; + Ok(scope_placement_validated_index( + &inputs.getattr("scope_index")?, + enforce_placement, + Some(PyTuple::new(py, usages)?), + )? + .into_bound(py)) +} + /// Python's `validate_scope_index`: raise one `ScopeValidationError` for its error diagnostics. #[pyfunction] #[pyo3(signature = (*, index))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs index eef36dc840..749b42b474 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs @@ -14,7 +14,7 @@ const OPEN_KINDS: [&str; 3] = [SOURCE_KIND, "ref", "dbt_ref"]; /// Partial-check reasons by model name; empty when SQL analysis is off. #[pyfunction] -fn semantic_coverage<'py>(project: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn semantic_coverage<'py>(project: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = project.py(); let reasons: Bound<'py, PyDict> = PyDict::new(py); if !project diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index 11779304c8..0c241bde84 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -19,7 +19,6 @@ ) ALLOWED_MACRO_LOAD_PATHS: frozenset[str] = frozenset( { - "src/sqlbuild/compiler/compile/main/sql_expansion_context.py", "src/sqlbuild/compiler/compile/_helpers/render/user_macros.py", } ) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8656b3341a..8eeeb6e185 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -71,6 +71,7 @@ from sqlbuild.compiler.compile.models import ( CompiledAudit, CompiledFunction, CompiledLineageColumnFact, + CompiledLineageSourceFact, CompiledModel, CompiledObjectKey, CompiledProject, @@ -2298,9 +2299,36 @@ def declared_relations( def externally_described_models( inputs: DiscoveredProjectInputs, model_schemas: dict[str, ModelSchemaDeclaration] ) -> list[Path]: ... -def resolve_required_columns( - query_sql: str, references: list[tuple[str, str]], dialect: str | None -) -> list[tuple[str, str, str]]: ... +def analyze_resolved_column_reads( + *, query_sql: str, references: Sequence[CompileSqlReference], dialect: str | None +) -> tuple[CompiledLineageSourceFact, ...]: ... +def build_effective_target_config( + *, discovered_inputs: DiscoveredProjectInputs, selected_target: str | None = None +) -> TargetConfig | None: ... +def build_effective_connection_config( + *, + discovered_inputs: DiscoveredProjectInputs, + selected_target: str | None = None, + cli_vars: dict[str, object] | None = None, +) -> dict[str, object]: ... +def build_effective_runtime_config( + *, + discovered_inputs: DiscoveredProjectInputs, + selected_target: str | None = None, + cli_vars: dict[str, object] | None = None, +) -> tuple[str | None, dict[str, object], str]: ... +def build_sql_expansion_context( + *, + project_dir: Path, + value_renderer: TypedSqlValueRenderer, + cli_vars: dict[str, object] | None = None, + discovered_inputs: DiscoveredProjectInputs | None = None, + declaration_scope: DeclarationScopeBuild | None = None, + static_declaration_scope: DeclarationScopeBuild | None = None, +) -> SqlExpansionContext: ... +def build_effective_settings_config( + *, discovered_inputs: DiscoveredProjectInputs +) -> SettingsConfig: ... def public_declaration_indexes( inputs: DiscoveredProjectInputs, ) -> tuple[dict[str, EnumDeclaration], dict[str, ConstantDeclaration]]: ... diff --git a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py index 5a1a66cd8e..15aebce087 100644 --- a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py @@ -6,7 +6,11 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import discover_project_inputs, validate_preview_target +from sqlbuild._native import ( + build_effective_settings_config, + discover_project_inputs, + validate_preview_target, +) from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( resolve_adapter_connection_context, ) @@ -19,7 +23,6 @@ build_command_progress_reporters, ) from sqlbuild.cli.progress.models import CommandProgressReporters -from sqlbuild.compiler.compile.main.effective_settings import build_effective_settings_config from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.presentation.main.supports_color import supports_color diff --git a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py index e48e6316ca..e09fa8cee4 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py @@ -373,7 +373,7 @@ def _rewrite_absolute_project_imports( return ast.Module(body=body, type_ignores=tree.type_ignores) -def _inventory_project_macros( +def inventory_project_macros( *, macro_files: tuple[DiscoveredMacroFile, ...] ) -> StaticMacroInventory: """Validate and inventory project macro exports without executing authored code.""" diff --git a/src/sqlbuild/compiler/compile/main/_analyze_columns_and_lineage.py b/src/sqlbuild/compiler/compile/main/_analyze_columns_and_lineage.py deleted file mode 100644 index bfae6b5bae..0000000000 --- a/src/sqlbuild/compiler/compile/main/_analyze_columns_and_lineage.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Public compile-time resolved column-read analysis operation.""" - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import ( - CompiledLineageSourceFact, - CompileSqlReference, -) - - -def analyze_resolved_column_reads( - *, - query_sql: str, - references: tuple[CompileSqlReference, ...], - dialect: str | None, -) -> tuple[CompiledLineageSourceFact, ...]: - """Return CTE-aware column reads resolved to external resources.""" - - reference_rows: list[tuple[str, str]] = [ - (str(reference.ref_kind), reference.ref_name) for reference in references - ] - reads: list[tuple[str, str, str]] = _native.resolve_required_columns( - query_sql, reference_rows, dialect - ) - return tuple( - CompiledLineageSourceFact( - resource_type=resource_type, resource_name=resource_name, column_name=column_name - ) - for resource_type, resource_name, column_name in reads - ) diff --git a/src/sqlbuild/compiler/compile/main/_effective_target.py b/src/sqlbuild/compiler/compile/main/_effective_target.py deleted file mode 100644 index bcbf17691f..0000000000 --- a/src/sqlbuild/compiler/compile/main/_effective_target.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Effective target config helpers for CLI entrypoints.""" - -from __future__ import annotations - -from sqlbuild._native import resolve_target_config, resolve_target_name -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import TargetConfig - - -def build_effective_target_config( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None = None, -) -> TargetConfig | None: - """Build effective target config without compiling resources.""" - - target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=selected_target, - ) - if target_name is None: - return None - return resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ) diff --git a/src/sqlbuild/compiler/compile/main/_effective_target_namespace.py b/src/sqlbuild/compiler/compile/main/_effective_target_namespace.py deleted file mode 100644 index 87184dda4a..0000000000 --- a/src/sqlbuild/compiler/compile/main/_effective_target_namespace.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Public effective target namespace operation.""" - -from sqlbuild._native import ( - build_effective_target_namespace as _build_effective_target_namespace, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import TargetConfig - - -def build_effective_target_namespace( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None, - connection_database: object | None, - connection_schema: object | None, - default_database: str | None, - default_schema: str | None, -) -> tuple[str | None, TargetConfig | None, str | None, str | None]: - """Resolve the target and its effective database/schema without compiling resources.""" - - return _build_effective_target_namespace( - discovered_inputs=discovered_inputs, - selected_target=selected_target, - connection_database=connection_database, - connection_schema=connection_schema, - default_database=default_database, - default_schema=default_schema, - ) diff --git a/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py b/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py deleted file mode 100644 index b19902120c..0000000000 --- a/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Shared compile-input projection into a usage-enriched scope index.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild._native import ( - get_placement_validated_scope_index, -) -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.scopes.models import ScopeIndex, UsageRecord - - -def scope_index_with_compile_usages(*, inputs: CompileInputsView) -> ScopeIndex: - """Return compile inputs' canonical index with all usage and placement facts.""" - - collected_usages: list[UsageRecord] = list( - _native.model_schema_scope_usages( - inputs.discovered_inputs.model_schema_files, - inputs.declaration_scope.resolver if inputs.declaration_scope is not None else None, - ) - ) - for model_input in inputs.model_inputs: - collected_usages.extend(model_input.macro_usages) - collected_usages.extend(model_input.declaration_usages) - for source_input in inputs.source_inputs: - collected_usages.extend(source_input.declaration_usages) - for function_input in inputs.sql_function_inputs: - collected_usages.extend(function_input.declaration_usages) - for audit_input in inputs.audit_inputs: - collected_usages.extend(audit_input.declaration_usages) - for test_input in inputs.test_inputs: - collected_usages.extend(test_input.declaration_usages) - for scenario_input in inputs.scenario_inputs: - collected_usages.extend(scenario_input.declaration_usages) - return get_placement_validated_scope_index( - index=inputs.scope_index, - enforce_placement=inputs.project_config.scopes.enforce_placement, - compile_usages=tuple(collected_usages), - ) diff --git a/src/sqlbuild/compiler/compile/main/_semantic_coverage.py b/src/sqlbuild/compiler/compile/main/_semantic_coverage.py deleted file mode 100644 index e960e20a53..0000000000 --- a/src/sqlbuild/compiler/compile/main/_semantic_coverage.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Public semantic-check coverage report.""" - -from sqlbuild._native import semantic_coverage -from sqlbuild.compiler.compile.types import CompiledProjectView - - -def get_semantic_coverage(project: CompiledProjectView) -> dict[str, tuple[str, ...]]: - """Return per-model partial-check reasons.""" - return semantic_coverage(project) diff --git a/src/sqlbuild/compiler/compile/main/_source_bindings.py b/src/sqlbuild/compiler/compile/main/_source_bindings.py deleted file mode 100644 index 083f3e1c5a..0000000000 --- a/src/sqlbuild/compiler/compile/main/_source_bindings.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Planner entrypoint for warehouse-backed semantic binding.""" - -from sqlbuild._native import source_binding_diagnostics -from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompilerDiagnostic -from sqlbuild.compiler.compile.types import CompiledProjectView - - -def get_source_binding_diagnostics( - *, - project: CompiledProjectView, - columns: dict[str, tuple[ColumnInfo, ...]], - profile: ExpressionInferenceProfile, - selected_keys: frozenset[CompiledObjectKey], -) -> tuple[CompilerDiagnostic, ...]: - """Bind selected source readers to already-inspected warehouse columns.""" - return source_binding_diagnostics( - project=project, columns=columns, profile=profile, selected_keys=selected_keys - ) diff --git a/src/sqlbuild/compiler/compile/main/_static_macro_inventory.py b/src/sqlbuild/compiler/compile/main/_static_macro_inventory.py deleted file mode 100644 index b261bc641c..0000000000 --- a/src/sqlbuild/compiler/compile/main/_static_macro_inventory.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Compiler static macro inventory without authored module execution.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile._helpers.render.user_macros import _inventory_project_macros -from sqlbuild.compiler.compile.models import StaticMacroInventory -from sqlbuild.compiler.discovery.models import DiscoveredMacroFile - - -def inventory_project_macros( - *, macro_files: tuple[DiscoveredMacroFile, ...] -) -> StaticMacroInventory: - """Return validated AST exports and faults without executing macro modules.""" - - return _inventory_project_macros(macro_files=macro_files) diff --git a/src/sqlbuild/compiler/compile/main/effective_config.py b/src/sqlbuild/compiler/compile/main/effective_config.py deleted file mode 100644 index dfb2777f87..0000000000 --- a/src/sqlbuild/compiler/compile/main/effective_config.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Effective project config helpers for compile and CLI entrypoints.""" - -from __future__ import annotations - -from sqlbuild._native import ( - build_effective_connection, - build_effective_vars, - resolve_target_config, - resolve_target_name, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import TargetConfig - - -def build_effective_connection_config( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None = None, - cli_vars: dict[str, object] | None = None, -) -> dict[str, object]: - """Build the effective project connection config without compiling resources.""" - - target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=selected_target, - ) - target_config: TargetConfig | None = None - if target_name is not None: - target_config = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ) - effective_vars: dict[str, object] = build_effective_vars( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_config=target_config, - cli_vars={} if cli_vars is None else cli_vars, - ) - return build_effective_connection( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_config=target_config, - effective_vars=effective_vars, - ) diff --git a/src/sqlbuild/compiler/compile/main/effective_runtime.py b/src/sqlbuild/compiler/compile/main/effective_runtime.py deleted file mode 100644 index 8f1a270910..0000000000 --- a/src/sqlbuild/compiler/compile/main/effective_runtime.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Effective runtime config helpers for CLI entrypoints.""" - -from __future__ import annotations - -from sqlbuild._native import ( - build_effective_vars, - resolve_run_id, - resolve_target_config, - resolve_target_name, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import TargetConfig - - -def build_effective_runtime_config( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None = None, - cli_vars: dict[str, object] | None = None, -) -> tuple[str | None, dict[str, object], str]: - """Build environment, vars, and invocation id without compiling resources.""" - - target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=selected_target, - ) - target_config: TargetConfig | None = None - if target_name is not None: - target_config = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ) - effective_vars: dict[str, object] = build_effective_vars( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_config=target_config, - cli_vars={} if cli_vars is None else cli_vars, - ) - return target_name, effective_vars, resolve_run_id(selected_run_id=None) diff --git a/src/sqlbuild/compiler/compile/main/effective_settings.py b/src/sqlbuild/compiler/compile/main/effective_settings.py deleted file mode 100644 index dbe8e63a9b..0000000000 --- a/src/sqlbuild/compiler/compile/main/effective_settings.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Effective project settings helper for compile and CLI entrypoints.""" - -from __future__ import annotations - -from sqlbuild._native import ( - build_effective_settings, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import SettingsConfig - - -def build_effective_settings_config( - *, discovered_inputs: DiscoveredProjectInputs -) -> SettingsConfig: - """Build effective settings without compiling resources.""" - - return build_effective_settings( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) diff --git a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py deleted file mode 100644 index b1ca22b3d8..0000000000 --- a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Build the context required to expand a project's authored SQL bodies.""" - -from __future__ import annotations - -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild._native import ( - build_declaration_scope, - build_effective_vars, - discover_project_inputs, - rebind_declaration_scope, - resolve_effective_adapter_name, - resolve_effective_collection_rendering, -) -from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros -from sqlbuild.compiler.compile.models import ( - ConstantDeclaration, - DeclarationResolutionContext, - DeclarationScopeBuild, - EnumDeclaration, - LoadedMacro, - MacroContext, - SqlExpansionContext, -) -from sqlbuild.compiler.compile.types import TypedSqlValueRenderer -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -def build_sql_expansion_context( - *, - project_dir: Path, - value_renderer: TypedSqlValueRenderer, - cli_vars: dict[str, object] | None = None, - discovered_inputs: DiscoveredProjectInputs | None = None, - declaration_scope: DeclarationScopeBuild | None = None, - static_declaration_scope: DeclarationScopeBuild | None = None, -) -> SqlExpansionContext: - """Assemble SQL expansion inputs; a static scope lends only its index to private macros.""" - - effective_discovered_inputs: DiscoveredProjectInputs = ( - discovered_inputs - if discovered_inputs is not None - else discover_project_inputs(project_dir=project_dir, sql_analysis_enabled_override=False) - ) - effective_vars: dict[str, object] = build_effective_vars( - project_config=effective_discovered_inputs.project_config, - local_config=effective_discovered_inputs.local_config, - target_config=None, - cli_vars={} if cli_vars is None else cli_vars, - ) - scope: DeclarationScopeBuild = declaration_scope or _build_declaration_scope( - discovered_inputs=effective_discovered_inputs, - static_scope=static_declaration_scope, - sql_lexical_syntax=value_renderer.sql_lexical_syntax, - ) - loaded_macros: dict[str, LoadedMacro] = scope.loaded_macros - enums: dict[str, EnumDeclaration] - constants: dict[str, ConstantDeclaration] - enums, constants = _native.public_declaration_indexes(effective_discovered_inputs) - local_declarations: dict[Path, DeclarationResolutionContext] = {} - for model_file in effective_discovered_inputs.model_files: - local_declarations[model_file.file_path] = DeclarationResolutionContext( - enums={declaration.name: declaration for declaration in model_file.enum_declarations}, - constants={ - declaration.name: declaration for declaration in model_file.constant_declarations - }, - ) - return SqlExpansionContext( - effective_vars=effective_vars, - loaded_macros=loaded_macros, - macro_context=MacroContext( - adapter_name=resolve_effective_adapter_name( - project_config=effective_discovered_inputs.project_config, - local_config=effective_discovered_inputs.local_config, - ), - sql_analysis_enabled=False, - target_name=effective_discovered_inputs.project_config.default_target, - vars=effective_vars, - _value_renderer=value_renderer, - _collection_rendering=resolve_effective_collection_rendering( - project_config=effective_discovered_inputs.project_config, - declaration_override=None, - ), - _enforce_explicit_references=( - effective_discovered_inputs.project_config.references.enforce_explicit - ), - ), - enums=enums, - constants=constants, - value_renderer=value_renderer, - collection_rendering=resolve_effective_collection_rendering( - project_config=effective_discovered_inputs.project_config, - declaration_override=None, - ), - local_declarations=local_declarations, - declaration_resolver=scope.resolver, - ) - - -def _build_declaration_scope( - *, - discovered_inputs: DiscoveredProjectInputs, - static_scope: DeclarationScopeBuild | None, - sql_lexical_syntax: SqlLexicalSyntax, -) -> DeclarationScopeBuild: - loaded_macros: dict[str, LoadedMacro] = load_project_macros(discovered_inputs.macro_files) - rebound: DeclarationScopeBuild | None = ( - None - if static_scope is None - else rebind_declaration_scope( - scope=static_scope, discovered_inputs=discovered_inputs, loaded_macros=loaded_macros - ) - ) - return rebound or build_declaration_scope( - discovered_inputs=discovered_inputs, - loaded_macros=loaded_macros, - sql_lexical_syntax=sql_lexical_syntax, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py b/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py index 139e393c53..1f1389a977 100644 --- a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py +++ b/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py @@ -8,10 +8,8 @@ from functools import lru_cache from typing import Any +from sqlbuild._native import analyze_resolved_column_reads from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.main._analyze_columns_and_lineage import ( - analyze_resolved_column_reads, -) from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, CompiledLineageSourceFact, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index 1bd31d8da6..c4bc2a6e0a 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -23,12 +23,12 @@ resolve_bounded_cursor_override, resolve_cursor_input_roles, resolve_effective_microbatch_batch_size, + source_binding_diagnostics, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._source_bindings import get_source_binding_diagnostics from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, @@ -386,7 +386,7 @@ def build_plan_entries( for diagnostic in project.diagnostics if diagnostic.is_error and diagnostic.resource_type != CompiledResourceType.SQL_TEST ), - *get_source_binding_diagnostics( + *source_binding_diagnostics( project=project, columns=relations.source_warehouse_columns, profile=adapter.expression_inference_profile(), diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py index 9a3614fde1..44a93ea5ed 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( ConnectionProgressReporter, apply_command_warehouse, + build_effective_connection_config, render_dbt_execution_summary_footer, render_dbt_interop_plan, resolve_target_config, @@ -16,7 +17,6 @@ build_connection_progress_reporter, ) from sqlbuild.cli.commands.models import DbtSqlbuildWorkContext -from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.integrations.dbt._helpers.pipeline.execute import execute_dbt_commands diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py index dd3a7ac299..6486e5cc97 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py @@ -10,10 +10,10 @@ from sqlbuild._native import ( apply_command_warehouse, + build_effective_connection_config, resolve_model_deferral_inputs, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.exceptions import PlannerInputError diff --git a/src/sqlbuild/lint/_helpers/expansion.py b/src/sqlbuild/lint/_helpers/expansion.py index 8bbbbeb57a..ea60fed18b 100644 --- a/src/sqlbuild/lint/_helpers/expansion.py +++ b/src/sqlbuild/lint/_helpers/expansion.py @@ -8,6 +8,7 @@ import sqlbuild._native as _native from sqlbuild._native import ( + build_sql_expansion_context, discover_project_inputs, expand_sql_with_spans, external_identifiers_for_headers, @@ -17,7 +18,6 @@ from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter from sqlbuild.compiler.compile.constants import MACRO_TOKEN from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main.sql_expansion_context import build_sql_expansion_context from sqlbuild.compiler.compile.models import ( CompiledSqlExpansion, DeclarationScopeBuild, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 87b1d933fb..08b31e3398 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -13,7 +13,6 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.render.user_macros as macros_module -import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module import sqlbuild.rule_engine._helpers.host.custom_host_pool as custom_host_pool_module import sqlbuild.rule_engine._helpers.run.rule_threads as rule_threads_module import sqlbuild.rule_engine._helpers.run.rules as rules_module @@ -834,7 +833,6 @@ def counting_build( ) monkeypatch.setattr(native_module, "build_declaration_scope", counting_build) - monkeypatch.setattr(expansion_context_module, "build_declaration_scope", counting_build) warm: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) @@ -874,9 +872,7 @@ def test_given_cold_scoped_macro_sql_test_when_compiling_then_early_lint_uses_pr scope_builds: list[Path] = [] macro_loads: list[dict[str, LoadedMacro]] = [] original_build: Callable[..., DeclarationScopeBuild] = native_module.build_declaration_scope - original_load: Callable[..., dict[str, LoadedMacro]] = ( - expansion_context_module.load_project_macros - ) + original_load: Callable[..., dict[str, LoadedMacro]] = macros_module.load_project_macros def counting_build( *, @@ -899,9 +895,7 @@ def recording_load(macro_files: tuple[DiscoveredMacroFile, ...]) -> dict[str, Lo return loaded monkeypatch.setattr(native_module, "build_declaration_scope", counting_build) - monkeypatch.setattr(expansion_context_module, "build_declaration_scope", counting_build) monkeypatch.setattr(macros_module, "load_project_macros", recording_load) - monkeypatch.setattr(expansion_context_module, "load_project_macros", recording_load) cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py index 9d44085150..b19e6a429c 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py @@ -6,12 +6,15 @@ import pytest -from sqlbuild._native import build_compiled_project, discover_project_inputs +from sqlbuild._native import ( + build_compiled_project, + discover_project_inputs, + source_binding_diagnostics, +) from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile._helpers.analysis import binding_requests -from sqlbuild.compiler.compile.main._source_bindings import get_source_binding_diagnostics from sqlbuild.compiler.compile.models import CompiledObjectKey, CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( @@ -123,19 +126,19 @@ def test_given_catalog_when_rebinding_physical_shapes_then_does_not_reuse_stale_ profile: ExpressionInferenceProfile = ExpressionInferenceProfile( sql_analysis_dialect="snowflake" ) - missing: tuple[CompilerDiagnostic, ...] = get_source_binding_diagnostics( + missing: tuple[CompilerDiagnostic, ...] = source_binding_diagnostics( project=project, columns={"orders": (ColumnInfo("customer", "VARCHAR"),)}, profile=profile, selected_keys=keys, ) - present: tuple[CompilerDiagnostic, ...] = get_source_binding_diagnostics( + present: tuple[CompilerDiagnostic, ...] = source_binding_diagnostics( project=project, columns={"orders": (ColumnInfo("OrderId", "INTEGER"),)}, profile=profile, selected_keys=keys, ) - repeated: tuple[CompilerDiagnostic, ...] = get_source_binding_diagnostics( + repeated: tuple[CompilerDiagnostic, ...] = source_binding_diagnostics( project=project, columns={"orders": (ColumnInfo("customer", "VARCHAR"),)}, profile=profile, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py index 56176ec05b..f6d98e54f3 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py @@ -8,11 +8,14 @@ import pytest import sqlbuild._native as native -from sqlbuild._native import build_compiled_project, discover_project_inputs +from sqlbuild._native import ( + build_compiled_project, + discover_project_inputs, + source_binding_diagnostics, +) from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile.main._source_bindings import get_source_binding_diagnostics from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( @@ -99,7 +102,7 @@ def test_given_effective_target_when_binding_quotes_then_honours_session_setting adapter=SnowflakeAdapter(), selected_target="selected", ) - physical: tuple[CompilerDiagnostic, ...] = get_source_binding_diagnostics( + physical: tuple[CompilerDiagnostic, ...] = source_binding_diagnostics( project=project, columns={"orders": (ColumnInfo(name="customer", type="VARCHAR"),)}, profile=ExpressionInferenceProfile(sql_analysis_dialect="snowflake"), diff --git a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py index 1cce92fb83..b4605751e6 100644 --- a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py +++ b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py @@ -291,7 +291,7 @@ def test_given_freshness_code_when_checking_sql_ownership_then_matches_contract( ), CustomRuleTestCase( description="compile input macro load passes", - path="src/sqlbuild/compiler/compile/main/sql_expansion_context.py", + path="src/sqlbuild/compiler/compile/_helpers/render/user_macros.py", source="loaded = load_project_macros(())\n", expected_fault_count=0, ), diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py index 13b69a940a..f310432175 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py @@ -99,7 +99,7 @@ def test_given_pending_run_when_showing_detail_then_persisted_summary_is_refresh lambda **kwargs: {"database": "ANALYTICS"}, ) monkeypatch.setattr( - "sqlbuild.compiler.compile.main._effective_target.build_effective_target_config", + "sqlbuild._native.build_effective_target_config", lambda **kwargs: SimpleNamespace(database=None), ) request: CostCommandRequest = CostCommandRequest( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py index a7a4ad428d..f07166bac9 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py @@ -4,9 +4,7 @@ import pytest -from sqlbuild.compiler.compile.main._analyze_columns_and_lineage import ( - analyze_resolved_column_reads, -) +from sqlbuild._native import analyze_resolved_column_reads from sqlbuild.compiler.compile.models import CompiledLineageSourceFact, CompileSqlReference from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.references.types import SqlReferenceKind diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py b/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py index 8f38b953d8..6617c86aa4 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import resolve_run_id -from sqlbuild.compiler.compile.main.effective_runtime import build_effective_runtime_config +from sqlbuild._native import build_effective_runtime_config, resolve_run_id from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.models import ( LocalConfig, diff --git a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py index 5577218e17..8dafbf4b74 100644 --- a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py +++ b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py @@ -8,7 +8,7 @@ import pytest -import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module +import sqlbuild._native as native_module import sqlbuild.compiler.planner._helpers.sql_tests.fixture_formatting as fixture_formatting_module from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -64,10 +64,10 @@ def test_given_value_renderer_when_building_expansion_context_then_scope_uses_it monkeypatch: pytest.MonkeyPatch, ) -> None: _ = write_fixture_format_project(tmp_path=tmp_path, fixture_sql="SELECT 1 AS id, TRUE AS flag") - build_scope: Mock = Mock(wraps=expansion_context_module.build_declaration_scope) - monkeypatch.setattr(expansion_context_module, "build_declaration_scope", build_scope) + build_scope: Mock = Mock(wraps=native_module.build_declaration_scope) + monkeypatch.setattr(native_module, "build_declaration_scope", build_scope) - _ = expansion_context_module.build_sql_expansion_context( + _ = native_module.build_sql_expansion_context( project_dir=tmp_path, value_renderer=_CustomSyntaxDuckDbAdapter() ) From 71d4a37f8a87d7575f08b7635050f4b6161ad960 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 05:44:49 +0100 Subject: [PATCH 30/40] chore: keep the custom-rule runtime guards as user code --- python-boundary.toml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index c23af68ebc..9162367a28 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 56777 +remaining_lines = 53504 [[stays]] path = "executor" @@ -304,6 +304,16 @@ path = "compiler/macro_bridge/_helpers/store_environment.py" reason = "user-code" note = "Fingerprints the Python interpreter and installed packages that project macros import from; the macro call store cache key." +[[stays]] +path = "rule_engine/classes/runtime_guard.py" +reason = "user-code" +note = "Guards custom-rule execution against side effects in user rule code." + +[[stays]] +path = "rule_engine/classes/module_state_monitor.py" +reason = "user-code" +note = "Watches user rule modules for state changes between custom-rule runs." + [[stays]] path = "compiler/compile/_helpers/render/user_macros.py" reason = "user-code" From 54c4a48cd85a5d72bde0ca6cc950f2f863305211 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:10:33 +0100 Subject: [PATCH 31/40] refactor: resolve planner strategy, audit scheduling and microbatch bounds natively --- crates/sqlbuild-analysis/src/lib.rs | 1 + .../_helpers/decimal_scale.rs | 36 ++ .../planner_strategy/_helpers/lineage_walk.rs | 42 ++ .../src/planner_strategy/_helpers/mod.rs | 8 + .../planner_strategy/_helpers/python_int.rs | 31 + .../src/planner_strategy/_helpers/reasons.rs | 30 + .../_helpers/schema_policy.rs | 11 + .../src/planner_strategy/_helpers/timeline.rs | 17 + .../planner_strategy/_helpers/type_names.rs | 27 + .../src/planner_strategy/_helpers/warnings.rs | 90 +++ .../src/planner_strategy/constants.rs | 78 +++ .../src/planner_strategy/errors.rs | 92 +++ .../main/cap_integer_bounds.rs | 20 + .../main/cap_timeline_bounds.rs | 33 + .../main/capped_watermark_input.rs | 71 +++ .../main/classify_cursor_sql_type.rs | 41 ++ .../main/count_integer_batches.rs | 21 + .../main/count_timeline_batches.rs | 25 + .../main/cursor_type_warning.rs | 60 ++ .../main/effective_run_scope.rs | 17 + .../main/lineage_reference.rs | 32 + .../main/materialization_type.rs | 15 + .../main/microbatch_limit_warning.rs | 23 + .../src/planner_strategy/main/mod.rs | 15 + .../main/model_plan_action.rs | 87 +++ .../planner_strategy/main/model_warnings.rs | 53 ++ .../planner_strategy/main/schema_actions.rs | 41 ++ .../main/singular_attachment.rs | 63 ++ .../src/planner_strategy/mod.rs | 11 + .../src/planner_strategy/models.rs | 180 ++++++ .../src/planner_strategy/tests.rs | 6 + .../src/planner_strategy/tests/helpers.rs | 115 ++++ .../tests/test_planner_strategy.rs | 585 ++++++++++++++++++ .../src/planner_strategy/tests/test_types.rs | 83 +++ .../src/planner_strategy/types.rs | 39 ++ .../src/type_system/main/mod.rs | 1 + .../type_system/main/polyglot_type_args.rs | 17 + .../src/bindings/_helpers/cursor/durations.rs | 25 +- .../src/bindings/_helpers/graph/algorithms.rs | 2 +- .../src/bindings/_helpers/graph/mod.rs | 2 +- .../src/bindings/_helpers/mod.rs | 1 + .../_helpers/model_config/config_values.rs | 5 +- .../_helpers/planner_strategy/audits.rs | 161 +++++ .../_helpers/planner_strategy/cursor_types.rs | 117 ++++ .../_helpers/planner_strategy/lineage.rs | 172 +++++ .../_helpers/planner_strategy/microbatches.rs | 324 ++++++++++ .../bindings/_helpers/planner_strategy/mod.rs | 7 + .../_helpers/planner_strategy/registration.rs | 15 + .../_helpers/planner_strategy/strategy.rs | 185 ++++++ .../_helpers/planner_strategy/values.rs | 168 +++++ .../src/bindings/main/register.rs | 7 +- src/sqlbuild/_native.pyi | 92 ++- .../_helpers/command_planning/static.py | 2 +- .../planner/_helpers/output/audit_entry.py | 13 +- .../_helpers/output/audit_scheduling.py | 173 ------ .../_helpers/output/cursor_type_check.py | 219 ------- .../_helpers/output/microbatch_bounds.py | 175 ------ .../_helpers/output/microbatch_count.py | 50 -- .../planner/_helpers/output/plan_entry.py | 40 +- .../planner/_helpers/output/strategy.py | 389 ------------ .../planner/_helpers/resolve/lineage.py | 49 -- .../compiler/planner/_helpers/reuse/clone.py | 2 +- .../planner/_helpers/warehouse/snapshot.py | 14 +- .../planner/main/execution/display_plan.py | 3 +- .../main/execution/microbatch_limit.py | 57 -- .../_helpers/materializations/microbatch.py | 4 +- .../_helpers/test_module_paths.py | 1 + .../planner/_helpers/changes/test_detect.py | 14 +- .../planner/_helpers/test_audit_scheduling.py | 10 +- .../_helpers/test_cursor_type_check.py | 4 +- .../planner/_helpers/test_strategy_action.py | 2 +- .../_helpers/test_strategy_schema_actions.py | 2 +- .../_helpers/test_strategy_warnings.py | 2 +- 73 files changed, 3434 insertions(+), 1191 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/_helpers/decimal_scale.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/_helpers/lineage_walk.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/_helpers/python_int.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/_helpers/reasons.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/_helpers/schema_policy.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/_helpers/timeline.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/_helpers/warnings.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/constants.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/errors.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/cap_integer_bounds.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/cap_timeline_bounds.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/capped_watermark_input.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/count_integer_batches.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/count_timeline_batches.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/effective_run_scope.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/lineage_reference.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/microbatch_limit_warning.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/model_warnings.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/schema_actions.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/singular_attachment.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/models.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/tests.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/types.rs create mode 100644 crates/sqlbuild-analysis/src/type_system/main/polyglot_type_args.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/output/audit_scheduling.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/output/cursor_type_check.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/output/microbatch_count.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/output/strategy.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/resolve/lineage.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/microbatch_limit.py diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 2a191262b2..383892f92e 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -20,6 +20,7 @@ pub mod lineage_command; pub mod lineage_graph; pub mod manifest; pub mod planner_changes; +pub mod planner_strategy; pub mod python_plan; pub mod python_selection; pub mod python_values; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/decimal_scale.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/decimal_scale.rs new file mode 100644 index 0000000000..c10f3ec64d --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/decimal_scale.rs @@ -0,0 +1,36 @@ +//! The scale of a decimal type written as `DECIMAL`, `NUMERIC` or `NUMBER`. + +use crate::planner_strategy::constants::DECIMAL_TYPE_NAMES; + +/// Digits after optional spaces, then optional spaces: the rest of the text. +fn digits_then_spaces(text: &str) -> Option<(&str, &str)> { + let text: &str = text.trim_start_matches(char::is_whitespace); + let end: usize = text + .find(|character: char| !character.is_ascii_digit()) + .unwrap_or(text.len()); + if end == 0 { + return None; + } + Some(( + &text[..end], + text[end..].trim_start_matches(char::is_whitespace), + )) +} + +/// Whether a full `DECIMAL[(p[, s])]` match has scale zero (absent counts as zero); `None` when the text does not match. +pub(crate) fn decimal_scale_is_zero(normalized: &str) -> Option { + let rest: &str = DECIMAL_TYPE_NAMES + .iter() + .find_map(|name| normalized.strip_prefix(name))?; + if rest.is_empty() { + return Some(true); + } + let inner: &str = rest.strip_prefix('(')?.strip_suffix(')')?; + let (_, after_precision) = digits_then_spaces(inner)?; + if after_precision.is_empty() { + return Some(true); + } + let (scale, tail) = digits_then_spaces(after_precision.strip_prefix(',')?)?; + tail.is_empty() + .then(|| scale.chars().all(|digit| digit == '0')) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/lineage_walk.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/lineage_walk.rs new file mode 100644 index 0000000000..04424283af --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/lineage_walk.rs @@ -0,0 +1,42 @@ +//! Breadth-first walks over SQL reference lineage. + +use std::collections::{HashSet, VecDeque}; + +use crate::planner_strategy::constants::{FUNCTION_REFERENCE_KINDS, MODEL_REFERENCE_KIND}; +use crate::planner_strategy::models::ReferenceIdentity; +use crate::planner_strategy::types::ReferenceGraph; + +/// The references a model or function reference reads, when the project has its target. +pub(crate) fn upstream_references( + graph: &G, + identity: &ReferenceIdentity, +) -> Result, G::Error> { + let references: Option> = if identity.kind == MODEL_REFERENCE_KIND { + graph.model_references(&identity.name)? + } else if FUNCTION_REFERENCE_KINDS.contains(&identity.kind.as_str()) { + graph.function_references(&identity.name)? + } else { + None + }; + Ok(references.unwrap_or_default()) +} + +/// A first-in, first-out queue of references that skips identities it has expanded. +pub(crate) struct LineageQueue { + pub(crate) pending: VecDeque, + pub(crate) visited: HashSet, +} + +impl LineageQueue { + pub(crate) fn new(roots: Vec) -> Self { + Self { + pending: roots.into(), + visited: HashSet::new(), + } + } + + /// Whether an identity is newly visited. + pub(crate) fn visit(&mut self, identity: &ReferenceIdentity) -> bool { + self.visited.insert(identity.clone()) + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/mod.rs new file mode 100644 index 0000000000..2aa0d6e824 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod decimal_scale; +pub(crate) mod lineage_walk; +pub(crate) mod python_int; +pub(crate) mod reasons; +pub(crate) mod schema_policy; +pub(crate) mod timeline; +pub(crate) mod type_names; +pub(crate) mod warnings; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/python_int.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/python_int.rs new file mode 100644 index 0000000000..b93cf5529e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/python_int.rs @@ -0,0 +1,31 @@ +//! Python's `int(text)` for batch sizes. + +use sqlbuild_core::text::main::decimal_digit_value::decimal_digit_value; + +/// Base-10 `int(text)`: surrounding whitespace, a sign, decimal digits and single underscores between them; `None` where Python raises or the value leaves `i128`. +pub(crate) fn python_int(text: &str) -> Option { + let trimmed: &str = text.trim(); + let (negative, digits) = match trimmed.chars().next() { + Some('-') => (true, &trimmed[1..]), + Some('+') => (false, &trimmed[1..]), + _ => (false, trimmed), + }; + let mut value: i128 = 0; + let mut previous_underscore: bool = true; + for character in digits.chars() { + if character == '_' { + if previous_underscore { + return None; + } + previous_underscore = true; + continue; + } + let digit: u32 = decimal_digit_value(character)?; + value = value.checked_mul(10)?.checked_add(i128::from(digit))?; + previous_underscore = false; + } + if previous_underscore { + return None; + } + Some(if negative { -value } else { value }) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/reasons.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/reasons.rs new file mode 100644 index 0000000000..01e1e5f61b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/reasons.rs @@ -0,0 +1,30 @@ +//! Plan reasons of a model's own change. + +use crate::planner_strategy::models::{ChangeFacts, PlanChangeKind, PlanReason}; + +/// The reason of a first run or a definition change; `None` for renames and no change. +pub(crate) fn own_change_reason(kind: PlanChangeKind) -> Option { + match kind { + PlanChangeKind::FirstRun => Some(PlanReason::FirstRun), + PlanChangeKind::QueryChanged => Some(PlanReason::QueryChanged), + PlanChangeKind::FunctionChanged => Some(PlanReason::FunctionChanged), + PlanChangeKind::ConfigChanged => Some(PlanReason::ConfigChanged), + PlanChangeKind::SchemaChanged => Some(PlanReason::SchemaChanged), + PlanChangeKind::Renamed | PlanChangeKind::NoChange => None, + } +} + +/// The reason a full backfill rebuilds the table. +pub(crate) fn backfill_reason(change: &ChangeFacts) -> PlanReason { + if change.query_changed { + PlanReason::QueryChanged + } else if !change.changed_functions.is_empty() { + PlanReason::FunctionChanged + } else if change.config_changed { + PlanReason::ConfigChanged + } else if !change.schema_findings.is_empty() { + PlanReason::SchemaChanged + } else { + PlanReason::FullRefresh + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/schema_policy.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/schema_policy.rs new file mode 100644 index 0000000000..ffd98fbd0e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/schema_policy.rs @@ -0,0 +1,11 @@ +//! The effective `on_schema_change` policy. + +use crate::planner_strategy::constants::APPEND_NEW_COLUMNS; + +/// `on_schema_change or append_new_columns`. +pub(crate) fn effective_on_schema_change(on_schema_change: Option<&str>) -> &str { + match on_schema_change { + Some(value) if !value.is_empty() => value, + _ => APPEND_NEW_COLUMNS, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/timeline.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/timeline.rs new file mode 100644 index 0000000000..6fc532bdd2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/timeline.rs @@ -0,0 +1,17 @@ +//! One clipped batch step on a caller's timeline. + +use crate::planner_strategy::types::Timeline; + +/// `min(moment + step, end)`, keeping `moment + step` on ties as Python's `min` does. +pub(crate) fn clipped_step( + timeline: &T, + moment: &T::Moment, + end: &T::Moment, +) -> Result { + let next: T::Moment = timeline.advance(moment)?; + Ok(if timeline.precedes(end, &next)? { + end.clone() + } else { + next + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs new file mode 100644 index 0000000000..aae5b546d0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs @@ -0,0 +1,27 @@ +//! Cursor types of warehouse type names and Polyglot type arguments. + +use serde_json::{Map, Value}; + +use crate::planner_strategy::constants::{INTEGER_TYPE_NAMES, TIMESTAMP_TYPE_NAMES}; +use crate::planner_strategy::models::CursorType; + +/// The cursor type of a known type name. +pub(crate) fn named_type(name: &str) -> Option { + if TIMESTAMP_TYPE_NAMES.contains(&name) { + return Some(CursorType::Timestamp); + } + INTEGER_TYPE_NAMES + .contains(&name) + .then_some(CursorType::Integer) +} + +/// Python's `str()` of one parsed type argument, upper-cased. +pub(crate) fn argument(args: &Map, key: &str) -> String { + match args.get(key) { + None => String::new(), + Some(Value::String(text)) => text.to_uppercase(), + Some(Value::Null) => "NONE".to_owned(), + Some(Value::Bool(flag)) => if *flag { "TRUE" } else { "FALSE" }.to_owned(), + Some(other) => other.to_string().to_uppercase(), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/warnings.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/warnings.rs new file mode 100644 index 0000000000..303e0b0bfd --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/warnings.rs @@ -0,0 +1,90 @@ +//! The schema-finding and forward-only replay warnings of one model. + +use crate::planner_changes::models::{SchemaChangeKind, SchemaColumnSource, SchemaFindingRow}; +use crate::planner_strategy::constants::QUERY_CHANGE_TRACKING_HELP; +use crate::planner_strategy::models::{ + ChangeFacts, MaterializationType, WarningInputs, WarningRow, WarningSeverity, +}; + +/// Python's `str()` of an optional type. +fn shown(value: Option<&String>) -> &str { + value.map_or("None", String::as_str) +} + +pub(crate) fn warning(severity: WarningSeverity, message: String) -> WarningRow { + WarningRow { + severity, + message, + code: None, + } +} + +/// The warning of one schema finding, if it has one. +pub(crate) fn finding_warning( + finding: &SchemaFindingRow, + type_enforcement: bool, +) -> Option { + let name: &str = &finding.column_name; + let expected: &str = shown(finding.expected_type.as_ref()); + let actual: &str = shown(finding.actual_type.as_ref()); + if type_enforcement && finding.source == SchemaColumnSource::Yml { + return (finding.kind == SchemaChangeKind::ColumnTypeChanged).then(|| { + warning( + WarningSeverity::Warning, + format!( + "enforced column {name} type mismatch: expected {expected}, warehouse has {actual}" + ), + ) + }); + } + if finding.source != SchemaColumnSource::SqlAnalysis { + return None; + } + let description: String = match finding.kind { + SchemaChangeKind::ColumnAdded => match finding.expected_type.as_deref() { + Some(expected) if !expected.is_empty() => { + format!("new column not in warehouse ({expected})") + } + _ => "new column not in warehouse".to_owned(), + }, + SchemaChangeKind::ColumnTypeChanged => { + format!("expected {expected}, warehouse has {actual}") + } + SchemaChangeKind::ColumnRemoved => return None, + }; + Some(warning( + WarningSeverity::Info, + format!("inferred column {name}: {description}"), + )) +} + +/// The forward-only replay warning of an incremental model whose definition changed. +pub(crate) fn forward_only_warning( + inputs: &WarningInputs, + change: &ChangeFacts, +) -> Option { + if inputs.materialization != MaterializationType::Incremental + || !(change.query_changed || !change.changed_functions.is_empty()) + || !change.backfill_forward_only + { + return None; + } + let model_name: &str = &inputs.model_name; + let subject: String = if change.query_changed { + "the model's SQL changed".to_owned() + } else { + format!("function {} changed", change.changed_functions.join(", ")) + }; + Some(warning( + WarningSeverity::Warning, + format!( + "{model_name}: {subject}, so already-built dates keep their old results - only new \ + dates will use the new query. To apply the change to existing data, choose how much \ + to reprocess: replay_on_change full (rebuild every date with the new SQL), \ + replay_on_change bounded- e.g. bounded-14d (reprocess just that window), \ + or leave it forward-only (existing dates as-is, new dates use the new query - the \ + default). If this model's output does not depend on the SQL that changed, this is \ + safe to ignore. {QUERY_CHANGE_TRACKING_HELP}" + ), + )) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/constants.rs b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs new file mode 100644 index 0000000000..9029fe4ae4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs @@ -0,0 +1,78 @@ +//! Planner strategy constants shared with the Python planner. + +use crate::planner_strategy::models::PlanAction; + +/// Error code of an incremental model without a strategy. +pub const MISSING_STRATEGY_ERROR_CODE: &str = "S201"; +/// Error code of an incremental model with an unknown strategy. +pub const UNKNOWN_STRATEGY_ERROR_CODE: &str = "S202"; +/// Error code of a watermark input outside the model's lineage. +pub const LINEAGE_INPUT_ERROR_CODE: &str = "S302"; +/// Warning code of a recorded relation that no longer exists. +pub const RECORDED_RELATION_MISSING_WARNING_CODE: &str = "RECORDED_RELATION_MISSING"; +/// Warning code of a model planning more batches than its limit. +pub const MICROBATCH_LIMIT_WARNING_CODE: &str = "MICROBATCH_LIMIT_EXCEEDED"; +/// The `on_schema_change` value of a model that leaves schema changes alone. +pub const IGNORE_SCHEMA_CHANGE: &str = "ignore"; +/// The `on_schema_change` value of a model that fails on schema changes. +pub const FAIL_SCHEMA_CHANGE: &str = "fail"; +/// The `on_schema_change` value that adds new columns; the default. +pub const APPEND_NEW_COLUMNS: &str = "append_new_columns"; +/// The `on_schema_change` value that adds, drops and retypes columns. +pub const SYNC_ALL_COLUMNS: &str = "sync_all_columns"; +/// Incremental strategies and the plan action of each. +pub const INCREMENTAL_STRATEGIES: [(&str, PlanAction); 3] = [ + ("append", PlanAction::IncrementalAppend), + ("delete_insert", PlanAction::IncrementalDeleteInsert), + ("merge", PlanAction::IncrementalMerge), +]; +/// How to silence forward-only change warnings. +pub const QUERY_CHANGE_TRACKING_HELP: &str = "To silence these warnings, set this in sqlbuild_project.toml:\n [settings]\n query_change_tracking = false"; +/// Reference kind of a model reference. +pub const MODEL_REFERENCE_KIND: &str = "ref"; +/// Reference kind of a source reference. +pub const SOURCE_REFERENCE_KIND: &str = "source"; +/// Reference kinds of SQL function calls. +pub const FUNCTION_REFERENCE_KINDS: [&str; 2] = ["udf", "table_fn"]; +/// Warehouse type names classified as timestamp cursors. +pub const TIMESTAMP_TYPE_NAMES: [&str; 13] = [ + "TIMESTAMP", + "TIMESTAMPNTZ", + "TIMESTAMPLTZ", + "TIMESTAMPTZ", + "TIMESTAMP_S", + "TIMESTAMP_MS", + "TIMESTAMP_NS", + "DATE", + "DATE32", + "DATETIME", + "DATETIME2", + "DATETIME64", + "SMALLDATETIME", +]; +/// Warehouse type names classified as integer cursors. +pub const INTEGER_TYPE_NAMES: [&str; 17] = [ + "INT", + "BIG_INT", + "BIGINT", + "SMALL_INT", + "SMALLINT", + "TINY_INT", + "TINYINT", + "MEDIUMINT", + "INT128", + "INT256", + "UINT", + "UBIGINT", + "USMALLINT", + "UTINYINT", + "UMEDIUMINT", + "UINT128", + "UINT256", +]; +/// Decimal type names whose zero-scale form is an integer cursor. +pub const DECIMAL_TYPE_NAMES: [&str; 3] = ["DECIMAL", "NUMERIC", "NUMBER"]; +/// The type Polyglot reports for a type it does not know. +pub const POLYGLOT_CUSTOM_TYPE_NAME: &str = "CUSTOM"; +/// The dialect cursor type classification parses unknown types with. +pub const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/errors.rs b/crates/sqlbuild-analysis/src/planner_strategy/errors.rs new file mode 100644 index 0000000000..2be7a8dd5b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/errors.rs @@ -0,0 +1,92 @@ +//! Planner strategy errors. + +use crate::planner_strategy::constants::{ + LINEAGE_INPUT_ERROR_CODE, MISSING_STRATEGY_ERROR_CODE, UNKNOWN_STRATEGY_ERROR_CODE, +}; + +/// An incremental model whose strategy is missing or unknown. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StrategyError { + Missing { model: String }, + Unknown { model: String, strategy: String }, +} + +impl StrategyError { + /// The planner input error message. + pub fn message(&self) -> String { + match self { + Self::Missing { model } => { + format!("incremental model '{model}' is missing required incremental_strategy") + } + Self::Unknown { model, strategy } => { + format!("incremental model '{model}' has unknown strategy '{strategy}'") + } + } + } + + /// The planner input error code. + pub fn code(&self) -> &'static str { + match self { + Self::Missing { .. } => MISSING_STRATEGY_ERROR_CODE, + Self::Unknown { .. } => UNKNOWN_STRATEGY_ERROR_CODE, + } + } +} + +/// A lineage walk that failed in the caller or did not find a watermark input. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum LineageFailure { + Graph(E), + NotInLineage { model: String, input: String }, +} + +impl LineageFailure { + /// The planner input error message of a missing input. + pub fn message(model: &str, input: &str) -> String { + format!( + "model '{model}': cursor_inputs watermark references '{input}', but it is not in the \ + model's upstream lineage" + ) + } + + /// The planner input error code of a missing input. + pub fn code() -> &'static str { + LINEAGE_INPUT_ERROR_CODE + } +} + +/// A watermark input served by a capped microbatch producer. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CappedWatermarkInput { + pub model: String, + pub input: String, + pub producer: String, +} + +impl CappedWatermarkInput { + /// The compile input error message. + pub fn message(&self) -> String { + let Self { + model, + input, + producer, + } = self; + if producer != input { + return format!( + "model '{model}' uses watermark input '{input}' derived from capped producer \ + '{producer}'; capped producers cannot serve as watermark inputs" + ); + } + format!( + "model '{model}' uses capped producer '{producer}' as a watermark input; capped \ + producers cannot serve as watermark inputs" + ) + } +} + +/// A warehouse type Polyglot could not parse, which classifies as no cursor type. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TypeParseFailure { + pub normalized_type: String, + pub message: String, +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/cap_integer_bounds.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/cap_integer_bounds.rs new file mode 100644 index 0000000000..f792d81e90 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/cap_integer_bounds.rs @@ -0,0 +1,20 @@ +//! An integer cursor range capped to its first or last batches. + +use crate::planner_strategy::_helpers::python_int::python_int; +use crate::planner_strategy::models::{CapDirection, CappedBound}; + +/// The moved bound, or `None` when the batch size is not a positive integer. +pub fn cap_integer_bounds( + (start, end): (i128, i128), + batch_size: &str, + max_batches: i128, + direction: CapDirection, +) -> Option> { + let size: i128 = python_int(batch_size).filter(|size| *size > 0)?; + if direction == CapDirection::FromStart { + return Some(CappedBound::End(end.min(start + size * max_batches))); + } + let batch_count: i128 = (end - start + size - 1).div_euclid(size).max(0); + let skipped: i128 = (batch_count - max_batches).max(0); + Some(CappedBound::Start(start + size * skipped)) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/cap_timeline_bounds.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/cap_timeline_bounds.rs new file mode 100644 index 0000000000..c5fa4065d6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/cap_timeline_bounds.rs @@ -0,0 +1,33 @@ +//! A timestamp cursor range capped to its first or last batches. + +use crate::planner_strategy::_helpers::timeline::clipped_step; +use crate::planner_strategy::models::{CapDirection, CappedBound}; +use crate::planner_strategy::types::Timeline; + +/// The new end after `max_batches` steps from the start, or the start of the last `max_batches` batches. +pub fn cap_timeline_bounds( + timeline: &T, + (start, end): (T::Moment, T::Moment), + max_batches: i128, + direction: CapDirection, +) -> Result, T::Error> { + if direction == CapDirection::FromStart { + let mut capped_end: T::Moment = start; + for _ in 0..max_batches.max(0) { + capped_end = clipped_step(timeline, &capped_end, &end)?; + } + return Ok(CappedBound::End(capped_end)); + } + let mut boundaries: Vec = vec![start]; + while let Some(last) = boundaries.last().cloned() { + if !timeline.precedes(&last, &end)? { + break; + } + boundaries.push(clipped_step(timeline, &last, &end)?); + } + let last_index: usize = boundaries.len().saturating_sub(1); + let kept: usize = usize::try_from(max_batches).unwrap_or(0).min(last_index); + Ok(CappedBound::Start( + boundaries.swap_remove(last_index - kept), + )) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/capped_watermark_input.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/capped_watermark_input.rs new file mode 100644 index 0000000000..ec4a447e31 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/capped_watermark_input.rs @@ -0,0 +1,71 @@ +//! Watermark inputs served by capped microbatch producers. + +use crate::planner_strategy::_helpers::lineage_walk::{LineageQueue, upstream_references}; +use crate::planner_strategy::constants::MODEL_REFERENCE_KIND; +use crate::planner_strategy::errors::{CappedWatermarkInput, LineageFailure}; +use crate::planner_strategy::main::lineage_reference::lineage_reference; +use crate::planner_strategy::models::{ProducerFacts, ReferenceIdentity}; +use crate::planner_strategy::types::ReferenceGraph; + +/// The first capped microbatch model upstream of a reference, following microbatch producers through their watermark inputs. +fn capped_producer( + graph: &G, + reference: G::Reference, +) -> Result, LineageFailure> { + let mut queue: LineageQueue = LineageQueue::new(vec![reference]); + while let Some(current) = queue.pending.pop_front() { + let identity: ReferenceIdentity = + graph.identity(¤t).map_err(LineageFailure::Graph)?; + if !queue.visit(&identity) { + continue; + } + if identity.kind == MODEL_REFERENCE_KIND { + let Some(roots) = graph + .model_references(&identity.name) + .map_err(LineageFailure::Graph)? + else { + continue; + }; + let facts: ProducerFacts = graph + .producer_facts(&identity.name) + .map_err(LineageFailure::Graph)?; + if facts.microbatch && facts.capped { + return Ok(Some(identity.name)); + } + if facts.microbatch { + for input in graph + .watermark_inputs(&identity.name) + .map_err(LineageFailure::Graph)? + { + let next: G::Reference = + lineage_reference(graph, (&identity.name, roots.clone()), &input)?; + queue.pending.push_back(next); + } + continue; + } + } + queue + .pending + .extend(upstream_references(graph, &identity).map_err(LineageFailure::Graph)?); + } + Ok(None) +} + +/// Python's `validate_capped_watermark_inputs` for a microbatch model with these references and watermark inputs: the first input a capped producer serves. +pub fn capped_watermark_input( + graph: &G, + (model, roots): (&str, Vec), + inputs: Vec, +) -> Result, LineageFailure> { + for input in inputs { + let reference: G::Reference = lineage_reference(graph, (model, roots.clone()), &input)?; + if let Some(producer) = capped_producer(graph, reference)? { + return Ok(Some(CappedWatermarkInput { + model: model.to_owned(), + input, + producer, + })); + } + } + Ok(None) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs new file mode 100644 index 0000000000..22dedf16bd --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs @@ -0,0 +1,41 @@ +//! The cursor type a warehouse SQL type supports. + +use serde_json::{Map, Value}; + +use crate::planner_strategy::_helpers::decimal_scale::decimal_scale_is_zero; +use crate::planner_strategy::_helpers::type_names::{argument, named_type}; +use crate::planner_strategy::constants::{GENERIC_DIALECT, POLYGLOT_CUSTOM_TYPE_NAME}; +use crate::planner_strategy::errors::TypeParseFailure; +use crate::planner_strategy::models::CursorType; +use crate::type_system::main::polyglot_type_args::polyglot_type_args; + +/// Timestamp or integer for supported cross-adapter types; `Err` when Polyglot cannot parse an unknown type. +pub fn classify_cursor_sql_type(sql_type: &str) -> Result, TypeParseFailure> { + let normalized: String = sql_type + .to_uppercase() + .split_whitespace() + .collect::>() + .join(" "); + if let Some(integral) = decimal_scale_is_zero(&normalized) { + return Ok(integral.then_some(CursorType::Integer)); + } + let base: String = normalized + .split('(') + .next() + .unwrap_or_default() + .trim() + .replace(' ', "_"); + if let Some(cursor_type) = named_type(&base) { + return Ok(Some(cursor_type)); + } + let args: Map = + polyglot_type_args(&normalized, GENERIC_DIALECT).map_err(|message| TypeParseFailure { + normalized_type: normalized.clone(), + message, + })?; + let mut type_name: String = argument(&args, "data_type"); + if type_name == POLYGLOT_CUSTOM_TYPE_NAME { + type_name = argument(&args, "name").replace('_', ""); + } + Ok(named_type(&type_name)) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/count_integer_batches.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/count_integer_batches.rs new file mode 100644 index 0000000000..f8ce1417aa --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/count_integer_batches.rs @@ -0,0 +1,21 @@ +//! The number of batches in an integer cursor range. + +use crate::planner_strategy::_helpers::python_int::python_int; + +/// Batches of `batch_size` from `start` up to `end`; equal bounds are one batch when asked. +pub fn count_integer_batches( + (start, end): (i128, i128), + batch_size: &str, + equal_bounds_are_batch: bool, +) -> i128 { + let Some(size) = python_int(batch_size) else { + return 0; + }; + if equal_bounds_are_batch && start == end { + return 1; + } + if size <= 0 || start >= end { + return 0; + } + (end - start + size - 1).div_euclid(size) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/count_timeline_batches.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/count_timeline_batches.rs new file mode 100644 index 0000000000..38981643f8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/count_timeline_batches.rs @@ -0,0 +1,25 @@ +//! The number of batches between two aligned moments. + +use crate::planner_strategy::_helpers::timeline::clipped_step; +use crate::planner_strategy::types::Timeline; + +/// Steps from `start` until `end`, the last one clipped; equal bounds are one batch when asked. +pub fn count_timeline_batches( + timeline: &T, + (start, end): (T::Moment, T::Moment), + equal_bounds_are_batch: bool, +) -> Result { + if equal_bounds_are_batch + && !timeline.precedes(&start, &end)? + && !timeline.precedes(&end, &start)? + { + return Ok(1); + } + let mut current: T::Moment = start; + let mut count: i128 = 0; + while timeline.precedes(¤t, &end)? { + current = clipped_step(timeline, ¤t, &end)?; + count += 1; + } + Ok(count) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs new file mode 100644 index 0000000000..d7b05b6719 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs @@ -0,0 +1,60 @@ +//! Whether a cursor column's warehouse type matches the declared cursor type. + +use crate::planner_strategy::errors::TypeParseFailure; +use crate::planner_strategy::main::classify_cursor_sql_type::classify_cursor_sql_type; +use crate::planner_strategy::models::{CursorType, CursorTypeCheck, WarningRow, WarningSeverity}; + +fn cursor_type_name(cursor_type: CursorType) -> &'static str { + match cursor_type { + CursorType::Timestamp => "timestamp", + CursorType::Integer => "integer", + } +} + +/// An error (with SQL analysis) or warning when the warehouse type is the other cursor type, and any Polyglot parse error that left it unclassified. +pub fn cursor_type_warning( + check: &CursorTypeCheck, +) -> (Option, Option) { + let (Some(column), Some(declared)) = (&check.cursor_column, &check.cursor_type) else { + return (None, None); + }; + let Some((_, warehouse_type)) = check + .warehouse_columns + .iter() + .find(|(name, _)| name.to_uppercase() == column.to_uppercase()) + else { + return (None, None); + }; + let declared: CursorType = match declared.as_str() { + "timestamp" => CursorType::Timestamp, + "integer" => CursorType::Integer, + _ => return (None, None), + }; + let detected: CursorType = match classify_cursor_sql_type(warehouse_type) { + Ok(Some(detected)) => detected, + Ok(None) => return (None, None), + Err(failure) => return (None, Some(failure)), + }; + if detected == declared { + return (None, None); + } + let (severity, verb) = if check.sql_analysis_enabled { + (WarningSeverity::Error, "is") + } else { + (WarningSeverity::Warning, "appears to be") + }; + let message: String = format!( + "cursor column '{column}' has warehouse type '{warehouse_type}' which {verb} {}, but \ + cursor_type is '{}'", + cursor_type_name(detected), + cursor_type_name(declared) + ); + ( + Some(WarningRow { + severity, + message, + code: None, + }), + None, + ) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/effective_run_scope.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/effective_run_scope.rs new file mode 100644 index 0000000000..9ffdb9128d --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/effective_run_scope.rs @@ -0,0 +1,17 @@ +//! The run scope an audit gets after degradation. + +use crate::planner_strategy::models::AuditRunScope; + +/// Delta-and-final scope only applies to audits attached to incremental or snapshot models. +pub fn effective_run_scope( + requested: AuditRunScope, + attached_materialization: Option<&str>, +) -> AuditRunScope { + if requested == AuditRunScope::Final { + return AuditRunScope::Final; + } + match attached_materialization { + Some("incremental" | "snapshot") => AuditRunScope::DeltaAndFinal, + _ => AuditRunScope::Final, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/lineage_reference.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/lineage_reference.rs new file mode 100644 index 0000000000..f3a7564272 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/lineage_reference.rs @@ -0,0 +1,32 @@ +//! The reference a watermark input names in a model's upstream lineage. + +use crate::planner_strategy::_helpers::lineage_walk::{LineageQueue, upstream_references}; +use crate::planner_strategy::errors::LineageFailure; +use crate::planner_strategy::models::ReferenceIdentity; +use crate::planner_strategy::types::ReferenceGraph; + +/// The first reference named `input`, breadth first from a model's references through model and SQL function references. +pub fn lineage_reference( + graph: &G, + (model, roots): (&str, Vec), + input: &str, +) -> Result> { + let mut queue: LineageQueue = LineageQueue::new(roots); + while let Some(reference) = queue.pending.pop_front() { + let identity: ReferenceIdentity = + graph.identity(&reference).map_err(LineageFailure::Graph)?; + if identity.name == input { + return Ok(reference); + } + if !queue.visit(&identity) { + continue; + } + queue + .pending + .extend(upstream_references(graph, &identity).map_err(LineageFailure::Graph)?); + } + Err(LineageFailure::NotInLineage { + model: model.to_owned(), + input: input.to_owned(), + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs new file mode 100644 index 0000000000..f45163514c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs @@ -0,0 +1,15 @@ +//! The materialization type of a model's `materialized` value. + +use crate::planner_strategy::models::MaterializationType; + +/// A table when unset; an unknown string is a custom materialization. +pub fn materialization_type(materialized: Option<&str>) -> MaterializationType { + match materialized { + None | Some("table") => MaterializationType::Table, + Some("view") => MaterializationType::View, + Some("incremental") => MaterializationType::Incremental, + Some("snapshot") => MaterializationType::Snapshot, + Some("seed") => MaterializationType::Seed, + Some(_) => MaterializationType::Custom, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/microbatch_limit_warning.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/microbatch_limit_warning.rs new file mode 100644 index 0000000000..c7b6dd3371 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/microbatch_limit_warning.rs @@ -0,0 +1,23 @@ +//! The message of a model planning more batches than its limit. + +/// `None` within the limit; `cap` says whether the action caps (rather than fails or warns). +pub fn microbatch_limit_warning( + model_name: &str, + (max_batches, batch_count): (Option, i128), + (action, cap): (&str, bool), +) -> Option { + let max_batches: i128 = max_batches.filter(|maximum| batch_count > *maximum)?; + let follow_up: String = if cap { + format!( + "Selected {max_batches} batches and deferred {}.", + batch_count - max_batches + ) + } else { + "Use --max-microbatches with an intentional invocation-specific value for this backfill." + .to_owned() + }; + Some(format!( + "MICROBATCH LIMIT EXCEEDED: model '{model_name}' planned {batch_count} batches, above \ + the per-model limit of {max_batches} (action={action}). {follow_up}" + )) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs new file mode 100644 index 0000000000..533eb51d7b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs @@ -0,0 +1,15 @@ +pub mod cap_integer_bounds; +pub mod cap_timeline_bounds; +pub mod capped_watermark_input; +pub mod classify_cursor_sql_type; +pub mod count_integer_batches; +pub mod count_timeline_batches; +pub mod cursor_type_warning; +pub mod effective_run_scope; +pub mod lineage_reference; +pub mod materialization_type; +pub mod microbatch_limit_warning; +pub mod model_plan_action; +pub mod model_warnings; +pub mod schema_actions; +pub mod singular_attachment; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs new file mode 100644 index 0000000000..437d66d599 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs @@ -0,0 +1,87 @@ +//! The plan action and reason of one model. + +use crate::planner_strategy::_helpers::reasons::{backfill_reason, own_change_reason}; +use crate::planner_strategy::constants::INCREMENTAL_STRATEGIES; +use crate::planner_strategy::errors::StrategyError; +use crate::planner_strategy::models::{ + ChangeFacts, MaterializationType, ModelStrategy, PlanAction, PlanChangeKind, PlanReason, +}; + +/// Python's `resolve_model_plan_action`: disabled models skip, then materialization decides. +pub fn model_plan_action( + model: &ModelStrategy, + change: &ChangeFacts, + full_refresh: bool, +) -> Result<(PlanAction, PlanReason), StrategyError> { + if model.disabled { + return Ok((PlanAction::Skip, PlanReason::Disabled)); + } + let refreshed = |fallback: PlanReason, renamed: bool| -> PlanReason { + if full_refresh { + return PlanReason::FullRefresh; + } + if renamed && change.kind == PlanChangeKind::Renamed { + return PlanReason::Renamed; + } + own_change_reason(change.kind).unwrap_or(fallback) + }; + match model.materialization { + MaterializationType::Custom => { + return Ok((PlanAction::Custom, refreshed(PlanReason::NoChange, false))); + } + MaterializationType::View => { + return Ok(( + PlanAction::CreateView, + refreshed(PlanReason::NoChange, true), + )); + } + MaterializationType::Snapshot => { + return Ok(( + PlanAction::Snapshot, + refreshed(PlanReason::NormalIncremental, false), + )); + } + MaterializationType::Table + | MaterializationType::Incremental + | MaterializationType::Seed => {} + } + if full_refresh { + return Ok((PlanAction::CreateTable, PlanReason::FullRefresh)); + } + if change.kind == PlanChangeKind::FirstRun { + return Ok((PlanAction::CreateTable, PlanReason::FirstRun)); + } + if change.kind == PlanChangeKind::Renamed { + return Ok((PlanAction::CreateTable, PlanReason::Renamed)); + } + if change.backfill_full { + return Ok((PlanAction::CreateTable, backfill_reason(change))); + } + let edited: Option = match change.kind { + PlanChangeKind::QueryChanged => Some(PlanReason::QueryChanged), + PlanChangeKind::FunctionChanged => Some(PlanReason::FunctionChanged), + PlanChangeKind::ConfigChanged => Some(PlanReason::ConfigChanged), + PlanChangeKind::SchemaChanged => Some(PlanReason::SchemaChanged), + PlanChangeKind::FirstRun | PlanChangeKind::Renamed | PlanChangeKind::NoChange => None, + }; + if model.materialization == MaterializationType::Table { + return Ok(( + PlanAction::CreateTable, + edited.unwrap_or(PlanReason::NoChange), + )); + } + let Some(strategy) = model.incremental_strategy.as_deref() else { + return Err(StrategyError::Missing { + model: model.name.clone(), + }); + }; + let reason: PlanReason = edited.unwrap_or(PlanReason::NormalIncremental); + INCREMENTAL_STRATEGIES + .iter() + .find(|(name, _)| *name == strategy) + .map(|(_, action)| (*action, reason)) + .ok_or_else(|| StrategyError::Unknown { + model: model.name.clone(), + strategy: strategy.to_owned(), + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/model_warnings.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/model_warnings.rs new file mode 100644 index 0000000000..4a32320b83 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/model_warnings.rs @@ -0,0 +1,53 @@ +//! The plan warnings of one model. + +use crate::planner_changes::models::SchemaChangeKind; +use crate::planner_strategy::_helpers::schema_policy::effective_on_schema_change; +use crate::planner_strategy::_helpers::warnings::{finding_warning, forward_only_warning, warning}; +use crate::planner_strategy::constants::{ + FAIL_SCHEMA_CHANGE, IGNORE_SCHEMA_CHANGE, RECORDED_RELATION_MISSING_WARNING_CODE, +}; +use crate::planner_strategy::models::{ChangeFacts, WarningInputs, WarningRow, WarningSeverity}; + +/// Python's `build_model_warnings`, in its order. +pub fn model_warnings(inputs: &WarningInputs, change: &ChangeFacts) -> Vec { + let effective: &str = effective_on_schema_change(inputs.on_schema_change.as_deref()); + let mut warnings: Vec = Vec::new(); + if change.recorded_build_relation_missing { + warnings.push(WarningRow { + severity: WarningSeverity::Warning, + message: format!( + "{}: SQLBuild state records a previous build, but the relation no longer exists \ + in the warehouse. It may have been dropped outside SQLBuild; planning a first \ + run to recreate it.", + inputs.model_name + ), + code: Some(RECORDED_RELATION_MISSING_WARNING_CODE), + }); + } + if effective == FAIL_SCHEMA_CHANGE && !change.schema_findings.is_empty() { + warnings.push(warning( + WarningSeverity::Error, + "schema change detected and on_schema_change is set to fail".to_owned(), + )); + } + warnings.extend( + change + .schema_findings + .iter() + .filter_map(|finding| finding_warning(finding, inputs.type_enforcement)), + ); + warnings.extend(forward_only_warning(inputs, change)); + let backfill_findings: bool = change.schema_findings.iter().any(|finding| { + matches!( + finding.kind, + SchemaChangeKind::ColumnAdded | SchemaChangeKind::ColumnTypeChanged + ) + }); + if backfill_findings && !inputs.has_schema_actions && effective == IGNORE_SCHEMA_CHANGE { + warnings.push(warning( + WarningSeverity::Info, + "schema change detected but on_schema_change is set to ignore".to_owned(), + )); + } + warnings +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/schema_actions.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/schema_actions.rs new file mode 100644 index 0000000000..88c7b50b55 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/schema_actions.rs @@ -0,0 +1,41 @@ +//! Schema actions resolved from a model's schema findings and `on_schema_change`. + +use crate::planner_changes::models::{SchemaChangeKind, SchemaFindingRow}; +use crate::planner_strategy::_helpers::schema_policy::effective_on_schema_change; +use crate::planner_strategy::constants::{APPEND_NEW_COLUMNS, SYNC_ALL_COLUMNS}; +use crate::planner_strategy::models::{SchemaActionKind, SchemaActionRow}; + +/// Added columns under `append_new_columns`; added, removed and retyped columns under `sync_all_columns`. +pub fn schema_actions( + findings: &[SchemaFindingRow], + on_schema_change: Option<&str>, +) -> Vec { + let effective: &str = effective_on_schema_change(on_schema_change); + let sync: bool = effective == SYNC_ALL_COLUMNS; + if !sync && effective != APPEND_NEW_COLUMNS { + return Vec::new(); + } + findings + .iter() + .filter_map(|finding| { + let (kind, column_type) = match finding.kind { + SchemaChangeKind::ColumnAdded => { + (SchemaActionKind::AddColumn, finding.expected_type.clone()) + } + SchemaChangeKind::ColumnRemoved if sync => (SchemaActionKind::DropColumn, None), + SchemaChangeKind::ColumnTypeChanged if sync => ( + SchemaActionKind::AlterColumnType, + finding.expected_type.clone(), + ), + SchemaChangeKind::ColumnRemoved | SchemaChangeKind::ColumnTypeChanged => { + return None; + } + }; + Some(SchemaActionRow { + kind, + column_name: finding.column_name.clone(), + column_type, + }) + }) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/singular_attachment.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/singular_attachment.rs new file mode 100644 index 0000000000..cf6063ec4c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/singular_attachment.rs @@ -0,0 +1,63 @@ +//! Where a singular audit attaches, from its references and the model graph. + +use crate::planner_strategy::models::AuditAttachmentKind; +use crate::planner_strategy::types::Reaches; + +/// Whether every other referenced model reaches `candidate`. +fn reached_by_all( + candidate: &str, + models: &[String], + reaches: &Reaches<'_, E>, +) -> Result { + for other in models { + if other != candidate && !reaches(other, candidate)? { + return Ok(false); + } + } + Ok(true) +} + +/// The single model every other referenced model reaches, if there is one. +fn latest_owner(models: &[String], reaches: &Reaches<'_, E>) -> Result, E> { + let mut candidates: Vec<&String> = Vec::new(); + for candidate in models { + if reached_by_all(candidate, models, reaches)? { + candidates.push(candidate); + } + } + if candidates.len() == 1 { + return Ok(Some(candidates[0].clone())); + } + if candidates.is_empty() { + return Ok(None); + } + candidates.sort(); + let mut deepest: &String = candidates[0]; + for other in &candidates[1..] { + if reaches(deepest, other)? { + deepest = other; + } + } + Ok(reached_by_all(deepest, models, reaches)?.then(|| deepest.clone())) +} + +/// Python's `_infer_singular_attachment` over the audit's model and source reference names. +pub fn singular_attachment( + model_references: &[String], + source_references: &[String], + reaches: &Reaches<'_, E>, +) -> Result<(AuditAttachmentKind, Option), E> { + if model_references.is_empty() { + return Ok(match source_references.iter().min() { + Some(source) => (AuditAttachmentKind::Source, Some(source.clone())), + None => (AuditAttachmentKind::End, None), + }); + } + if let [model] = model_references { + return Ok((AuditAttachmentKind::Model, Some(model.clone()))); + } + Ok(match latest_owner(model_references, reaches)? { + Some(owner) => (AuditAttachmentKind::Model, Some(owner)), + None => (AuditAttachmentKind::End, None), + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/mod.rs b/crates/sqlbuild-analysis/src/planner_strategy/mod.rs new file mode 100644 index 0000000000..e7bd10bd20 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/mod.rs @@ -0,0 +1,11 @@ +//! Planner strategy: plan actions and reasons, schema actions, model warnings, audit attachment and run scope, cursor type checks, watermark lineage and microbatch counting and capping. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/models.rs b/crates/sqlbuild-analysis/src/planner_strategy/models.rs new file mode 100644 index 0000000000..5553b77357 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/models.rs @@ -0,0 +1,180 @@ +//! Planner strategy inputs and decisions. + +use crate::planner_changes::models::SchemaFindingRow; + +/// How a model is materialized; an unknown `materialized` value is a custom materialization. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MaterializationType { + View, + Table, + Incremental, + Snapshot, + Seed, + Custom, +} + +/// What the plan does with a model. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PlanAction { + CreateView, + CreateTable, + IncrementalAppend, + IncrementalDeleteInsert, + IncrementalMerge, + Snapshot, + Skip, + Custom, +} + +/// Why the plan does it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PlanReason { + FirstRun, + Renamed, + FullRefresh, + QueryChanged, + FunctionChanged, + ConfigChanged, + SchemaChanged, + NormalIncremental, + NoChange, + Disabled, +} + +/// How a model changed since its recorded build, including first runs and renames. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PlanChangeKind { + FirstRun, + Renamed, + QueryChanged, + FunctionChanged, + ConfigChanged, + SchemaChanged, + NoChange, +} + +/// The parts of a model's change result the strategy reads. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ChangeFacts { + pub kind: PlanChangeKind, + pub backfill_full: bool, + pub backfill_forward_only: bool, + pub query_changed: bool, + pub changed_functions: Vec, + pub config_changed: bool, + pub schema_findings: Vec, + pub recorded_build_relation_missing: bool, +} + +/// The model configuration the plan action depends on. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ModelStrategy { + pub name: String, + pub disabled: bool, + pub materialization: MaterializationType, + /// The `incremental_strategy` value when it is a string. + pub incremental_strategy: Option, +} + +/// What a schema action does to one column. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SchemaActionKind { + AddColumn, + DropColumn, + AlterColumnType, +} + +/// One schema change to apply to a target table. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SchemaActionRow { + pub kind: SchemaActionKind, + pub column_name: String, + pub column_type: Option, +} + +/// How severe a plan warning is. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WarningSeverity { + Error, + Warning, + Info, +} + +/// One plan warning about a model. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WarningRow { + pub severity: WarningSeverity, + pub message: String, + pub code: Option<&'static str>, +} + +/// The inputs of a model's plan warnings besides its change facts. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WarningInputs { + pub model_name: String, + pub materialization: MaterializationType, + pub has_schema_actions: bool, + pub on_schema_change: Option, + pub type_enforcement: bool, +} + +/// A cursor type. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CursorType { + Timestamp, + Integer, +} + +/// A model's cursor column and the warehouse columns its type is checked against. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CursorTypeCheck { + pub model_name: String, + pub cursor_column: Option, + pub cursor_type: Option, + pub warehouse_columns: Vec<(String, String)>, + pub sql_analysis_enabled: bool, +} + +/// Where an audit attaches. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AuditAttachmentKind { + Source, + Seed, + Model, + End, +} + +/// When an audit runs. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AuditRunScope { + Final, + DeltaAndFinal, +} + +/// The kind and name of one SQL reference. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct ReferenceIdentity { + pub kind: String, + pub name: String, +} + +/// The batch-limit facts of an upstream model. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProducerFacts { + pub microbatch: bool, + pub capped: bool, +} + +/// The end a microbatch cap keeps. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CapDirection { + FromStart, + FromEnd, +} + +/// The bound a cap moves: a new end when capping from the start, a new start when capping from the end. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CappedBound { + End(T), + Start(T), +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests.rs new file mode 100644 index 0000000000..30e1350d0c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_planner_strategy.rs"] +mod planner_strategy; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/helpers.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/helpers.rs new file mode 100644 index 0000000000..6d2c4b6c41 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/helpers.rs @@ -0,0 +1,115 @@ +use std::collections::BTreeMap; + +use crate::planner_changes::models::{SchemaChangeKind, SchemaColumnSource, SchemaFindingRow}; +use crate::planner_strategy::errors::LineageFailure; +use crate::planner_strategy::models::{ + ChangeFacts, PlanChangeKind, ProducerFacts, ReferenceIdentity, WarningRow, WarningSeverity, +}; +use crate::planner_strategy::tests::test_types::{Lineage, Steps}; +use crate::planner_strategy::types::{ReferenceGraph, Timeline}; + +pub(super) fn change(kind: PlanChangeKind) -> ChangeFacts { + ChangeFacts { + kind, + backfill_full: false, + backfill_forward_only: true, + query_changed: kind == PlanChangeKind::QueryChanged, + changed_functions: Vec::new(), + config_changed: false, + schema_findings: Vec::new(), + recorded_build_relation_missing: false, + } +} + +pub(super) fn finding( + kind: SchemaChangeKind, + source: SchemaColumnSource, + (expected_type, actual_type): (Option<&str>, Option<&str>), +) -> SchemaFindingRow { + SchemaFindingRow { + kind, + column_name: "amount".to_owned(), + source, + expected_type: expected_type.map(str::to_owned), + actual_type: actual_type.map(str::to_owned), + } +} + +pub(super) fn warning(severity: WarningSeverity, message: &str) -> WarningRow { + WarningRow { + severity, + message: message.to_owned(), + code: None, + } +} + +pub(super) fn names(items: &[&str]) -> Vec { + items.iter().map(|item| (*item).to_owned()).collect() +} + +impl Timeline for Steps { + type Moment = i64; + type Error = (); + + fn advance(&self, moment: &i64) -> Result { + Ok(moment + 3) + } + + fn precedes(&self, left: &i64, right: &i64) -> Result { + Ok(left < right) + } +} + +fn reference(kind: &str, name: &str) -> ReferenceIdentity { + ReferenceIdentity { + kind: kind.to_owned(), + name: name.to_owned(), + } +} + +impl Lineage { + pub(super) fn roots() -> Vec { + vec![reference("ref", "customers"), reference("udf", "discount")] + } +} + +impl ReferenceGraph for Lineage { + type Reference = ReferenceIdentity; + type Error = (); + + fn identity(&self, reference: &ReferenceIdentity) -> Result { + Ok(reference.clone()) + } + + fn model_references(&self, name: &str) -> Result>, ()> { + let models: BTreeMap<&str, Vec> = BTreeMap::from([ + ("customers", vec![reference("ref", "events")]), + ("events", vec![reference("source", "raw_events")]), + ("products", Vec::new()), + ]); + Ok(models.get(name).cloned()) + } + + fn function_references(&self, name: &str) -> Result>, ()> { + Ok((name == "discount").then(|| vec![reference("ref", "products")])) + } + + fn producer_facts(&self, name: &str) -> Result { + Ok(ProducerFacts { + microbatch: name == "customers" || self.capped.contains(name), + capped: self.capped.contains(name), + }) + } + + fn watermark_inputs(&self, name: &str) -> Result, ()> { + Ok(if name == "customers" { + names(&["events"]) + } else { + Vec::new() + }) + } +} + +pub(super) fn failure_text(failure: LineageFailure<()>) -> String { + format!("{failure:?}") +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs new file mode 100644 index 0000000000..a65170fcaf --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs @@ -0,0 +1,585 @@ +use std::collections::BTreeSet; + +use crate::planner_changes::models::{SchemaChangeKind, SchemaColumnSource}; +use crate::planner_strategy::constants::QUERY_CHANGE_TRACKING_HELP; +use crate::planner_strategy::main::cap_integer_bounds::cap_integer_bounds; +use crate::planner_strategy::main::cap_timeline_bounds::cap_timeline_bounds; +use crate::planner_strategy::main::capped_watermark_input::capped_watermark_input; +use crate::planner_strategy::main::classify_cursor_sql_type::classify_cursor_sql_type; +use crate::planner_strategy::main::count_integer_batches::count_integer_batches; +use crate::planner_strategy::main::count_timeline_batches::count_timeline_batches; +use crate::planner_strategy::main::lineage_reference::lineage_reference; +use crate::planner_strategy::main::microbatch_limit_warning::microbatch_limit_warning; +use crate::planner_strategy::main::model_plan_action::model_plan_action; +use crate::planner_strategy::main::model_warnings::model_warnings; +use crate::planner_strategy::main::schema_actions::schema_actions; +use crate::planner_strategy::main::singular_attachment::singular_attachment; +use crate::planner_strategy::models::{ + AuditAttachmentKind, CapDirection, CappedBound, ChangeFacts, CursorType, MaterializationType, + ModelStrategy, PlanAction, PlanChangeKind, PlanReason, SchemaActionKind, SchemaActionRow, + WarningInputs, WarningSeverity, +}; +use crate::planner_strategy::tests::helpers::{change, failure_text, finding, names, warning}; +use crate::planner_strategy::tests::test_types::{ + ActionTestCase, AttachmentTestCase, ClassifyTestCase, IntegerBatchTestCase, + LimitWarningTestCase, Lineage, LineageTestCase, SchemaActionTestCase, Steps, + TimelineBatchTestCase, WarningTestCase, +}; + +#[test] +fn given_model_change_when_resolving_action_then_matches_python_strategy() { + let test_cases = [ + ActionTestCase { + description: "view rename", + materialization: MaterializationType::View, + strategy: None, + change: change(PlanChangeKind::Renamed), + full_refresh: false, + expected_action: Ok((PlanAction::CreateView, PlanReason::Renamed)), + }, + ActionTestCase { + description: "snapshot without change", + materialization: MaterializationType::Snapshot, + strategy: None, + change: change(PlanChangeKind::NoChange), + full_refresh: false, + expected_action: Ok((PlanAction::Snapshot, PlanReason::NormalIncremental)), + }, + ActionTestCase { + description: "custom full refresh", + materialization: MaterializationType::Custom, + strategy: None, + change: change(PlanChangeKind::Renamed), + full_refresh: true, + expected_action: Ok((PlanAction::Custom, PlanReason::FullRefresh)), + }, + ActionTestCase { + description: "table query change", + materialization: MaterializationType::Table, + strategy: None, + change: change(PlanChangeKind::QueryChanged), + full_refresh: false, + expected_action: Ok((PlanAction::CreateTable, PlanReason::QueryChanged)), + }, + ActionTestCase { + description: "incremental full backfill of a config change", + materialization: MaterializationType::Incremental, + strategy: Some("merge"), + change: ChangeFacts { + backfill_full: true, + config_changed: true, + ..change(PlanChangeKind::ConfigChanged) + }, + full_refresh: false, + expected_action: Ok((PlanAction::CreateTable, PlanReason::ConfigChanged)), + }, + ActionTestCase { + description: "incremental append", + materialization: MaterializationType::Incremental, + strategy: Some("append"), + change: change(PlanChangeKind::NoChange), + full_refresh: false, + expected_action: Ok((PlanAction::IncrementalAppend, PlanReason::NormalIncremental)), + }, + ActionTestCase { + description: "incremental without strategy", + materialization: MaterializationType::Incremental, + strategy: None, + change: change(PlanChangeKind::NoChange), + full_refresh: false, + expected_action: Err( + "S201 incremental model 'orders' is missing required incremental_strategy" + .to_owned(), + ), + }, + ActionTestCase { + description: "incremental with unknown strategy", + materialization: MaterializationType::Incremental, + strategy: Some("upsert"), + change: change(PlanChangeKind::QueryChanged), + full_refresh: false, + expected_action: Err( + "S202 incremental model 'orders' has unknown strategy 'upsert'".to_owned(), + ), + }, + ]; + for test_case in test_cases { + let model = ModelStrategy { + name: "orders".to_owned(), + disabled: false, + materialization: test_case.materialization, + incremental_strategy: test_case.strategy.map(str::to_owned), + }; + assert_eq!( + model_plan_action(&model, &test_case.change, test_case.full_refresh) + .map_err(|error| format!("{} {}", error.code(), error.message())), + test_case.expected_action, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_schema_findings_when_resolving_actions_then_follows_on_schema_change() { + let added = SchemaActionRow { + kind: SchemaActionKind::AddColumn, + column_name: "amount".to_owned(), + column_type: Some("INTEGER".to_owned()), + }; + let test_cases = [ + SchemaActionTestCase { + description: "default appends new columns", + on_schema_change: None, + expected_actions: vec![added.clone()], + }, + SchemaActionTestCase { + description: "sync drops and retypes", + on_schema_change: Some("sync_all_columns"), + expected_actions: vec![ + added, + SchemaActionRow { + kind: SchemaActionKind::DropColumn, + column_name: "amount".to_owned(), + column_type: None, + }, + SchemaActionRow { + kind: SchemaActionKind::AlterColumnType, + column_name: "amount".to_owned(), + column_type: Some("BIGINT".to_owned()), + }, + ], + }, + SchemaActionTestCase { + description: "fail resolves nothing", + on_schema_change: Some("fail"), + expected_actions: Vec::new(), + }, + ]; + let findings = [ + finding( + SchemaChangeKind::ColumnAdded, + SchemaColumnSource::Yml, + (Some("INTEGER"), None), + ), + finding( + SchemaChangeKind::ColumnRemoved, + SchemaColumnSource::Yml, + (None, Some("INTEGER")), + ), + finding( + SchemaChangeKind::ColumnTypeChanged, + SchemaColumnSource::Yml, + (Some("BIGINT"), Some("INTEGER")), + ), + ]; + for test_case in test_cases { + assert_eq!( + schema_actions(&findings, test_case.on_schema_change), + test_case.expected_actions, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_change_facts_when_building_warnings_then_matches_python_messages() { + let test_cases = [ + WarningTestCase { + description: "ignored inferred addition", + on_schema_change: Some("ignore"), + type_enforcement: false, + change: ChangeFacts { + schema_findings: vec![finding( + SchemaChangeKind::ColumnAdded, + SchemaColumnSource::SqlAnalysis, + (Some(""), None), + )], + ..change(PlanChangeKind::SchemaChanged) + }, + expected_warnings: vec![ + warning( + WarningSeverity::Info, + "inferred column amount: new column not in warehouse", + ), + warning( + WarningSeverity::Info, + "schema change detected but on_schema_change is set to ignore", + ), + ], + }, + WarningTestCase { + description: "failing enforced type change", + on_schema_change: Some("fail"), + type_enforcement: true, + change: ChangeFacts { + schema_findings: vec![finding( + SchemaChangeKind::ColumnTypeChanged, + SchemaColumnSource::Yml, + (Some("BIGINT"), None), + )], + ..change(PlanChangeKind::SchemaChanged) + }, + expected_warnings: vec![ + warning( + WarningSeverity::Error, + "schema change detected and on_schema_change is set to fail", + ), + warning( + WarningSeverity::Warning, + "enforced column amount type mismatch: expected BIGINT, warehouse has None", + ), + ], + }, + WarningTestCase { + description: "forward-only function change", + on_schema_change: None, + type_enforcement: false, + change: ChangeFacts { + changed_functions: names(&["discount", "tax"]), + ..change(PlanChangeKind::FunctionChanged) + }, + expected_warnings: vec![warning( + WarningSeverity::Warning, + &format!( + "orders: function discount, tax changed, so already-built dates keep their \ + old results - only new dates will use the new query. To apply the change \ + to existing data, choose how much to reprocess: replay_on_change full \ + (rebuild every date with the new SQL), replay_on_change bounded- \ + e.g. bounded-14d (reprocess just that window), or leave it forward-only \ + (existing dates as-is, new dates use the new query - the default). If this \ + model's output does not depend on the SQL that changed, this is safe to \ + ignore. {QUERY_CHANGE_TRACKING_HELP}" + ), + )], + }, + ]; + for test_case in test_cases { + let inputs = WarningInputs { + model_name: "orders".to_owned(), + materialization: MaterializationType::Incremental, + has_schema_actions: false, + on_schema_change: test_case.on_schema_change.map(str::to_owned), + type_enforcement: test_case.type_enforcement, + }; + assert_eq!( + model_warnings(&inputs, &test_case.change), + test_case.expected_warnings, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_warehouse_type_when_classifying_cursor_type_then_matches_python() { + let test_cases = [ + ClassifyTestCase { + description: "timestamp with time zone", + sql_type: "timestamp with time zone", + expected_cursor_type: Some(CursorType::Timestamp), + }, + ClassifyTestCase { + description: "number(38, 0)", + sql_type: "number(38, 0)", + expected_cursor_type: Some(CursorType::Integer), + }, + ClassifyTestCase { + description: "NUMERIC(10,2)", + sql_type: "NUMERIC(10,2)", + expected_cursor_type: None, + }, + ClassifyTestCase { + description: "decimal", + sql_type: "decimal", + expected_cursor_type: Some(CursorType::Integer), + }, + ClassifyTestCase { + description: " big int ", + sql_type: " big int ", + expected_cursor_type: Some(CursorType::Integer), + }, + ClassifyTestCase { + description: "VARCHAR(20)", + sql_type: "VARCHAR(20)", + expected_cursor_type: None, + }, + ClassifyTestCase { + description: "INTEGER", + sql_type: "INTEGER", + expected_cursor_type: Some(CursorType::Integer), + }, + ]; + for test_case in test_cases { + assert_eq!( + classify_cursor_sql_type(test_case.sql_type).ok().flatten(), + test_case.expected_cursor_type, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_integer_range_when_counting_and_capping_then_matches_python() { + let test_cases = [ + IntegerBatchTestCase { + description: "partial last batch from the end", + bounds: (0, 10), + batch_size: " 3 ", + direction: CapDirection::FromEnd, + expected_count: 4, + expected_cap: Some(CappedBound::Start(6)), + }, + IntegerBatchTestCase { + description: "from the start", + bounds: (0, 10), + batch_size: "1_0", + direction: CapDirection::FromStart, + expected_count: 1, + expected_cap: Some(CappedBound::End(10)), + }, + IntegerBatchTestCase { + description: "unparsable size", + bounds: (0, 10), + batch_size: "1d", + direction: CapDirection::FromStart, + expected_count: 0, + expected_cap: None, + }, + IntegerBatchTestCase { + description: "zero size", + bounds: (0, 10), + batch_size: "0", + direction: CapDirection::FromEnd, + expected_count: 0, + expected_cap: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + count_integer_batches(test_case.bounds, test_case.batch_size, false), + test_case.expected_count, + "{}", + test_case.description + ); + assert_eq!( + cap_integer_bounds( + test_case.bounds, + test_case.batch_size, + 2, + test_case.direction + ), + test_case.expected_cap, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_timeline_range_when_counting_and_capping_then_clips_the_last_batch() { + let test_cases = [ + TimelineBatchTestCase { + description: "last two of four batches", + bounds: (0, 10), + max_batches: 2, + direction: CapDirection::FromEnd, + expected_count: 4, + expected_cap: CappedBound::Start(6), + }, + TimelineBatchTestCase { + description: "first two batches", + bounds: (0, 10), + max_batches: 2, + direction: CapDirection::FromStart, + expected_count: 4, + expected_cap: CappedBound::End(6), + }, + TimelineBatchTestCase { + description: "more allowed than planned", + bounds: (0, 4), + max_batches: 5, + direction: CapDirection::FromStart, + expected_count: 2, + expected_cap: CappedBound::End(4), + }, + TimelineBatchTestCase { + description: "equal bounds", + bounds: (5, 5), + max_batches: 1, + direction: CapDirection::FromEnd, + expected_count: 1, + expected_cap: CappedBound::Start(5), + }, + ]; + for test_case in test_cases { + assert_eq!( + count_timeline_batches(&Steps, test_case.bounds, true), + Ok(test_case.expected_count), + "{}", + test_case.description + ); + assert_eq!( + cap_timeline_bounds( + &Steps, + test_case.bounds, + test_case.max_batches, + test_case.direction + ), + Ok(test_case.expected_cap), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_microbatch_limit_when_exceeded_then_formats_python_warning() { + let test_cases = [ + LimitWarningTestCase { + description: "within the limit", + limit: (Some(2), 2), + action: ("warn", false), + expected_warning: None, + }, + LimitWarningTestCase { + description: "no limit", + limit: (None, 9), + action: ("error", false), + expected_warning: None, + }, + LimitWarningTestCase { + description: "capped", + limit: (Some(2), 5), + action: ("cap_from_end", true), + expected_warning: Some( + "MICROBATCH LIMIT EXCEEDED: model 'orders' planned 5 batches, above the \ + per-model limit of 2 (action=cap_from_end). Selected 2 batches and deferred 3.", + ), + }, + ]; + for test_case in test_cases { + assert_eq!( + microbatch_limit_warning("orders", test_case.limit, test_case.action).as_deref(), + test_case.expected_warning, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_audit_references_when_inferring_attachment_then_picks_latest_owner() { + let test_cases = [ + AttachmentTestCase { + description: "sources only", + models: Vec::new(), + sources: names(&["payments", "events"]), + expected_attachment: (AuditAttachmentKind::Source, Some("events".to_owned())), + }, + AttachmentTestCase { + description: "nothing", + models: Vec::new(), + sources: Vec::new(), + expected_attachment: (AuditAttachmentKind::End, None), + }, + AttachmentTestCase { + description: "downstream model owns", + models: names(&["orders", "customers"]), + sources: Vec::new(), + expected_attachment: (AuditAttachmentKind::Model, Some("orders".to_owned())), + }, + AttachmentTestCase { + description: "duplicate reference", + models: names(&["products", "products"]), + sources: Vec::new(), + expected_attachment: (AuditAttachmentKind::Model, Some("products".to_owned())), + }, + AttachmentTestCase { + description: "unrelated models", + models: names(&["products", "customers"]), + sources: Vec::new(), + expected_attachment: (AuditAttachmentKind::End, None), + }, + ]; + for test_case in test_cases { + let reaches = |from: &str, to: &str| -> Result { + Ok(from == "customers" && to == "orders") + }; + assert_eq!( + singular_attachment(&test_case.models, &test_case.sources, &reaches), + Ok(test_case.expected_attachment), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_watermark_input_when_walking_lineage_then_finds_reference_and_capped_producer() { + let test_cases = [ + LineageTestCase { + description: "through a function", + input: "products", + capped: &[], + expected_reference: Ok("products".to_owned()), + expected_capped: Ok(None), + }, + LineageTestCase { + description: "capped direct producer", + input: "customers", + capped: &["customers"], + expected_reference: Ok("customers".to_owned()), + expected_capped: Ok(Some("customers".to_owned())), + }, + LineageTestCase { + description: "capped producer behind a microbatch watermark", + input: "customers", + capped: &["events"], + expected_reference: Ok("customers".to_owned()), + expected_capped: Ok(Some("events".to_owned())), + }, + LineageTestCase { + description: "missing input", + input: "returns", + capped: &[], + expected_reference: Err( + "NotInLineage { model: \"orders\", input: \"returns\" }".to_owned() + ), + expected_capped: Err( + "NotInLineage { model: \"orders\", input: \"returns\" }".to_owned() + ), + }, + ]; + for test_case in test_cases { + let graph = Lineage { + capped: test_case + .capped + .iter() + .map(|name| (*name).to_owned()) + .collect::>(), + }; + assert_eq!( + lineage_reference(&graph, ("orders", Lineage::roots()), test_case.input) + .map(|reference| reference.name) + .map_err(failure_text), + test_case.expected_reference, + "{}", + test_case.description + ); + assert_eq!( + capped_watermark_input( + &graph, + ("orders", Lineage::roots()), + names(&[test_case.input]) + ) + .map(|found| found.map(|capped| capped.producer)) + .map_err(failure_text), + test_case.expected_capped, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs new file mode 100644 index 0000000000..e28e6bc03a --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs @@ -0,0 +1,83 @@ +use std::collections::BTreeSet; + +use crate::planner_strategy::models::{ + AuditAttachmentKind, CapDirection, CappedBound, ChangeFacts, CursorType, MaterializationType, + PlanAction, PlanReason, SchemaActionRow, WarningRow, +}; + +pub(super) struct ActionTestCase { + pub(super) description: &'static str, + pub(super) materialization: MaterializationType, + pub(super) strategy: Option<&'static str>, + pub(super) change: ChangeFacts, + pub(super) full_refresh: bool, + pub(super) expected_action: Result<(PlanAction, PlanReason), String>, +} + +pub(super) struct SchemaActionTestCase { + pub(super) description: &'static str, + pub(super) on_schema_change: Option<&'static str>, + pub(super) expected_actions: Vec, +} + +pub(super) struct WarningTestCase { + pub(super) description: &'static str, + pub(super) on_schema_change: Option<&'static str>, + pub(super) type_enforcement: bool, + pub(super) change: ChangeFacts, + pub(super) expected_warnings: Vec, +} + +pub(super) struct ClassifyTestCase { + pub(super) description: &'static str, + pub(super) sql_type: &'static str, + pub(super) expected_cursor_type: Option, +} + +pub(super) struct IntegerBatchTestCase { + pub(super) description: &'static str, + pub(super) bounds: (i128, i128), + pub(super) batch_size: &'static str, + pub(super) direction: CapDirection, + pub(super) expected_count: i128, + pub(super) expected_cap: Option>, +} + +pub(super) struct TimelineBatchTestCase { + pub(super) description: &'static str, + pub(super) bounds: (i64, i64), + pub(super) max_batches: i128, + pub(super) direction: CapDirection, + pub(super) expected_count: i128, + pub(super) expected_cap: CappedBound, +} + +pub(super) struct AttachmentTestCase { + pub(super) description: &'static str, + pub(super) models: Vec, + pub(super) sources: Vec, + pub(super) expected_attachment: (AuditAttachmentKind, Option), +} + +pub(super) struct LineageTestCase { + pub(super) description: &'static str, + pub(super) input: &'static str, + pub(super) capped: &'static [&'static str], + pub(super) expected_reference: Result, + pub(super) expected_capped: Result, String>, +} + +pub(super) struct LimitWarningTestCase { + pub(super) description: &'static str, + pub(super) limit: (Option, i128), + pub(super) action: (&'static str, bool), + pub(super) expected_warning: Option<&'static str>, +} + +/// Whole numbers stepping by three. +pub(super) struct Steps; + +/// `orders` reads `customers` and the `discount` function; `discount` reads `products`; `customers` is a microbatch model reading `events` as its watermark input. +pub(super) struct Lineage { + pub(super) capped: BTreeSet, +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/types.rs b/crates/sqlbuild-analysis/src/planner_strategy/types.rs new file mode 100644 index 0000000000..a8f5860b0d --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/types.rs @@ -0,0 +1,39 @@ +//! Planner strategy callback types. + +use crate::planner_strategy::models::{ProducerFacts, ReferenceIdentity}; + +/// Whether the second model name is downstream of the first. +pub type Reaches<'a, E> = dyn Fn(&str, &str) -> Result + 'a; + +/// A timeline of cursor moments the caller owns: one batch step and the ordering. +pub trait Timeline { + type Moment: Clone; + type Error; + + /// The moment one batch after `moment`. + fn advance(&self, moment: &Self::Moment) -> Result; + + /// Whether `left` sorts before `right`. + fn precedes(&self, left: &Self::Moment, right: &Self::Moment) -> Result; +} + +/// The project's SQL references as the caller holds them. +pub trait ReferenceGraph { + type Reference: Clone; + type Error; + + /// The kind and name of a reference. + fn identity(&self, reference: &Self::Reference) -> Result; + + /// The references of a model, when the project has it. + fn model_references(&self, name: &str) -> Result>, Self::Error>; + + /// The references of a SQL function, when the project has it. + fn function_references(&self, name: &str) -> Result>, Self::Error>; + + /// Whether a model is a microbatch model and caps its batches. + fn producer_facts(&self, name: &str) -> Result; + + /// The watermark input names of a model. + fn watermark_inputs(&self, name: &str) -> Result, Self::Error>; +} diff --git a/crates/sqlbuild-analysis/src/type_system/main/mod.rs b/crates/sqlbuild-analysis/src/type_system/main/mod.rs index 81be6727b9..85cc669961 100644 --- a/crates/sqlbuild-analysis/src/type_system/main/mod.rs +++ b/crates/sqlbuild-analysis/src/type_system/main/mod.rs @@ -1 +1,2 @@ pub mod normalize_type; +pub mod polyglot_type_args; diff --git a/crates/sqlbuild-analysis/src/type_system/main/polyglot_type_args.rs b/crates/sqlbuild-analysis/src/type_system/main/polyglot_type_args.rs new file mode 100644 index 0000000000..972ac1f6c0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/main/polyglot_type_args.rs @@ -0,0 +1,17 @@ +//! The scalar arguments of a type as Polyglot parses it. + +use polyglot_sql::{DataType, Dialect, DialectType}; +use serde_json::{Map, Value}; + +use crate::type_system::_helpers::parsed::parse_type; +use crate::type_system::_helpers::shallow_args::shallow_args; + +/// `parse_data_type(type_sql, dialect=...).args` scalars, or the parse error text. +pub fn polyglot_type_args(type_sql: &str, dialect: &str) -> Result, String> { + let polyglot: Dialect = dialect + .parse::() + .map(Dialect::get) + .map_err(|_| format!("unknown dialect '{dialect}'"))?; + let parsed: DataType = parse_type(type_sql, &polyglot)?; + Ok(shallow_args(&parsed)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs index a31d7399ca..7bd4590a5c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs @@ -52,12 +52,10 @@ fn duration_span(duration: &Bound<'_, PyAny>) -> PyResult { }) } -/// `moment` moved forward (`direction` 1) or back (`direction` -1) by `duration`. -#[pyfunction] -#[pyo3(signature = (*, moment, duration, direction))] -fn shift_by_cursor_duration<'py>( +/// `moment` (a `date` or `datetime`) moved by `span` in `direction`, keeping its type and `tzinfo`. +pub(crate) fn shifted_moment<'py>( moment: &Bound<'py, PyAny>, - duration: &Bound<'py, PyAny>, + span: &DurationSpan, direction: i64, ) -> PyResult> { let py: Python<'py> = moment.py(); @@ -71,11 +69,7 @@ fn shift_by_cursor_duration<'py>( } else { Moment::Date(civil_date(moment)?) }; - let shifted: Moment = session.checked(shift_by_duration( - typed, - &duration_span(duration)?, - direction, - ))?; + let shifted: Moment = session.checked(shift_by_duration(typed, span, direction))?; match shifted { Moment::Timestamp { local, fold } => replaced(moment, &local, fold), Moment::Date(date) => call_with( @@ -89,6 +83,17 @@ fn shift_by_cursor_duration<'py>( } } +/// `moment` moved forward (`direction` 1) or back (`direction` -1) by `duration`. +#[pyfunction] +#[pyo3(signature = (*, moment, duration, direction))] +fn shift_by_cursor_duration<'py>( + moment: &Bound<'py, PyAny>, + duration: &Bound<'py, PyAny>, + direction: i64, +) -> PyResult> { + shifted_moment(moment, &duration_span(duration)?, direction) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(parse_cursor_duration, module)?)?; module.add_function(wrap_pyfunction!(shift_by_cursor_duration, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs index ce588a9c1e..54c8e6f8f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs @@ -11,7 +11,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; const BUILTINS_MODULE: &str = "builtins"; /// Keys within `max_depth` edges of any start, plus the starts when `include_starts`. -fn closure<'py>( +pub(crate) fn closure<'py>( starts: Vec>, edges: &Bound<'py, PyAny>, (include_starts, max_depth): (bool, Option), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs index 5e3f7e22ae..63db7ad9d8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs @@ -1,4 +1,4 @@ -mod algorithms; +pub(crate) mod algorithms; mod execution_gates; pub(crate) mod keys; pub(crate) mod project_graph; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index d8ddc827ef..71b7bb0094 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -45,6 +45,7 @@ pub(crate) mod manifest; pub(crate) mod model_config; pub(crate) mod plan_output; pub(crate) mod planner_changes; +pub(crate) mod planner_strategy; pub(crate) mod presentation_text; pub(crate) mod project_assembly; pub(crate) mod python_values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs index ddae3fc318..26f364aa21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs @@ -38,7 +38,10 @@ fn wrong_type(key: &str, expected: &str, value: &Bound<'_, PyAny>) -> PyErr { /// An authored string, preserving a present empty string. #[pyfunction(name = "get_config_str")] #[pyo3(signature = (*, values, key))] -fn config_str<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult>> { +pub(crate) fn config_str<'py>( + values: &Bound<'py, PyAny>, + key: &str, +) -> PyResult>> { let Some(value) = authored(values, key)? else { return Ok(None); }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs new file mode 100644 index 0000000000..fbf692a524 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs @@ -0,0 +1,161 @@ +//! Audit attachment, target lineage and run scope. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PySet, PySetMethods, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_strategy::constants::{MODEL_REFERENCE_KIND, SOURCE_REFERENCE_KIND}; +use sqlbuild_analysis::planner_strategy::main::effective_run_scope::effective_run_scope; +use sqlbuild_analysis::planner_strategy::main::singular_attachment::singular_attachment; +use sqlbuild_analysis::planner_strategy::models::{AuditAttachmentKind, AuditRunScope}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::graph::algorithms::closure; +use crate::bindings::_helpers::planner_strategy::values::{ + PLANNER_EXCEPTIONS_MODULE, input_error, optional_string, text, +}; + +const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const MODEL_RESOURCE_TYPE: &str = "model"; +const FINAL_RUN_SCOPE: &str = "final"; + +/// `CompiledObjectKey(resource_type=CompiledResourceType(kind), name=name)`. +fn object_key<'py>(py: Python<'py>, kind: &str, name: &str) -> PyResult> { + call_with( + &imported(py, COMPILE_MODELS_MODULE, "CompiledObjectKey")?, + &[ + ( + "resource_type", + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.call1((kind,))?, + ), + ("name", PyString::new(py, name).into_any()), + ], + ) +} + +/// The names of an audit's references of one kind, in order. +fn reference_names(audit: &Bound<'_, PyAny>, kind: &str) -> PyResult> { + let mut names: Vec = Vec::new(); + for reference in audit.getattr("references")?.try_iter()? { + let reference: Bound<'_, PyAny> = reference?; + if text(&reference.getattr("ref_kind")?)? == kind { + names.push(text(&reference.getattr("ref_name")?)?); + } + } + Ok(names) +} + +fn attachment_value(kind: AuditAttachmentKind) -> &'static str { + match kind { + AuditAttachmentKind::Source => "source", + AuditAttachmentKind::Seed => "seed", + AuditAttachmentKind::Model => "model", + AuditAttachmentKind::End => "end", + } +} + +/// Audit attachment kind and attached target name; attached audits gate their target. +#[pyfunction] +#[pyo3(signature = (*, audit, downstream_deps))] +fn resolve_attachment_kind<'py>( + audit: &Bound<'py, PyAny>, + downstream_deps: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = audit.py(); + let attached_kind: Bound<'py, PyAny> = audit.getattr("attached_target_kind")?; + let (kind, name): (String, Option) = if attached_kind.is_none() { + let reaches = |from: &str, to: &str| -> PyResult { + let start: Bound<'py, PyAny> = object_key(py, MODEL_RESOURCE_TYPE, from)?; + closure(vec![start], downstream_deps, (false, None))?.contains(object_key( + py, + MODEL_RESOURCE_TYPE, + to, + )?) + }; + let (kind, name) = singular_attachment( + &reference_names(audit, MODEL_REFERENCE_KIND)?, + &reference_names(audit, SOURCE_REFERENCE_KIND)?, + &reaches, + )?; + (attachment_value(kind).to_owned(), name) + } else { + let name: Bound<'py, PyAny> = audit.getattr("attached_target_name")?; + if name.is_none() { + return Err(input_error( + py, + (PLANNER_EXCEPTIONS_MODULE, "PlannerInputError"), + &format!( + "audit '{}': attached audit is missing an attached target name", + text(&audit.getattr("name")?)? + ), + None, + )); + } + let kind: String = text( + &imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")? + .call1((attached_kind,))?, + )?; + (kind, Some(text(&name)?)) + }; + PyTuple::new( + py, + [ + imported(py, AUDITING_TYPES_MODULE, "AuditAttachmentKind")?.call1((kind,))?, + name.into_pyobject(py)?.into_any(), + ], + ) +} + +/// Whether an attached audit reads anything besides its target and the target's upstream. +#[pyfunction] +#[pyo3(signature = (*, audit, upstream_deps))] +fn reads_outside_target_lineage( + audit: &Bound<'_, PyAny>, + upstream_deps: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = audit.py(); + let kind: Bound<'_, PyAny> = audit.getattr("attached_target_kind")?; + let name: Bound<'_, PyAny> = audit.getattr("attached_target_name")?; + if kind.is_none() || name.is_none() { + return Ok(false); + } + let kind: String = + text(&imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind,))?)?; + let target: Bound<'_, PyAny> = object_key(py, &kind, &text(&name)?)?; + let lineage: Bound<'_, PySet> = closure(vec![target], upstream_deps, (true, None))?; + for dependency in audit.getattr("scope_deps")?.try_iter()? { + if !lineage.contains(dependency?)? { + return Ok(true); + } + } + Ok(false) +} + +/// The run scope after degradation: delta-and-final only for incremental and snapshot targets. +#[pyfunction] +#[pyo3(signature = (*, requested_run_scope, attached_model_materialization))] +fn resolve_effective_run_scope<'py>( + requested_run_scope: &Bound<'py, PyAny>, + attached_model_materialization: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = requested_run_scope.py(); + let requested: AuditRunScope = if text(requested_run_scope)? == FINAL_RUN_SCOPE { + AuditRunScope::Final + } else { + AuditRunScope::DeltaAndFinal + }; + let materialization: Option = optional_string(attached_model_materialization)?; + let scope: &str = match effective_run_scope(requested, materialization.as_deref()) { + AuditRunScope::Final => FINAL_RUN_SCOPE, + AuditRunScope::DeltaAndFinal => "delta_and_final", + }; + imported(py, AUDITING_TYPES_MODULE, "AuditRunScope")?.call1((scope,)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_attachment_kind, module)?)?; + module.add_function(wrap_pyfunction!(reads_outside_target_lineage, module)?)?; + module.add_function(wrap_pyfunction!(resolve_effective_run_scope, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs new file mode 100644 index 0000000000..848cb748f2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs @@ -0,0 +1,117 @@ +//! Warehouse cursor column type checks. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_strategy::errors::TypeParseFailure; +use sqlbuild_analysis::planner_strategy::main::classify_cursor_sql_type::classify_cursor_sql_type as classify; +use sqlbuild_analysis::planner_strategy::main::cursor_type_warning::cursor_type_warning; +use sqlbuild_analysis::planner_strategy::models::{CursorType, CursorTypeCheck}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_changes::results::planner_enum; +use crate::bindings::_helpers::planner_strategy::values::{optional_string, plan_warning, text}; + +const DEBUG_EVENT_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; +const PLANNER_LOGGER: &str = "sqlbuild.planner"; + +/// Log the Polyglot parse error that left a warehouse type unclassified. +fn log_parse_failure(py: Python<'_>, failure: &TypeParseFailure) -> PyResult<()> { + let text = |value: &str| PyString::new(py, value).into_any(); + call_with( + &imported(py, DEBUG_EVENT_MODULE, "log_debug_event")?, + &[ + ( + "logger", + imported(py, "logging", "getLogger")?.call1((PLANNER_LOGGER,))?, + ), + ( + "message", + text("cursor type classification parse failed; falling back"), + ), + ("sqlbuild_warehouse_type", text(&failure.normalized_type)), + ("sqlbuild_error", text(&failure.message)), + ], + ) + .map(|_| ()) +} + +/// The `CursorType` a supported cross-adapter SQL type holds, or `None`. +#[pyfunction] +#[pyo3(signature = (sql_type))] +fn classify_cursor_sql_type<'py>( + py: Python<'py>, + sql_type: &str, +) -> PyResult>> { + let detected: Option = match classify(sql_type) { + Ok(detected) => detected, + Err(failure) => { + log_parse_failure(py, &failure)?; + None + } + }; + detected + .map(|cursor_type| { + let value: &str = match cursor_type { + CursorType::Timestamp => "timestamp", + CursorType::Integer => "integer", + }; + planner_enum(py, "CursorType", value) + }) + .transpose() +} + +/// A `PlanWarning` when the warehouse cursor column type contradicts `cursor_type`. +#[pyfunction] +#[pyo3(signature = (**options))] +fn check_cursor_type_consistency<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult>> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "check_cursor_type_consistency", + options, + ( + &[ + "model_name", + "cursor_column", + "cursor_type", + "warehouse_columns", + "sql_analysis_enabled", + ], + &[], + ), + )?; + let model_name: Bound<'py, PyAny> = arguments.value("model_name")?; + let mut warehouse_columns: Vec<(String, String)> = Vec::new(); + for column in arguments.value("warehouse_columns")?.try_iter()? { + let column: Bound<'py, PyAny> = column?; + warehouse_columns.push(( + text(&column.getattr("name")?)?, + text(&column.getattr("type")?)?, + )); + } + let check = CursorTypeCheck { + model_name: text(&model_name)?, + cursor_column: optional_string(&arguments.value("cursor_column")?)?, + cursor_type: optional_string(&arguments.value("cursor_type")?)?, + warehouse_columns, + sql_analysis_enabled: arguments.flag("sql_analysis_enabled")?, + }; + let (warning, parse_failure) = cursor_type_warning(&check); + if let Some(failure) = parse_failure { + log_parse_failure(py, &failure)?; + } + warning + .map(|row| plan_warning(&model_name, row)) + .transpose() +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(classify_cursor_sql_type, module)?)?; + module.add_function(wrap_pyfunction!(check_cursor_type_consistency, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs new file mode 100644 index 0000000000..47bd1cb509 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs @@ -0,0 +1,172 @@ +//! Watermark inputs resolved through SQL reference lineage, and capped producers behind them. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_strategy::errors::LineageFailure; +use sqlbuild_analysis::planner_strategy::main::capped_watermark_input::capped_watermark_input; +use sqlbuild_analysis::planner_strategy::main::lineage_reference::lineage_reference; +use sqlbuild_analysis::planner_strategy::models::{ProducerFacts, ReferenceIdentity}; +use sqlbuild_analysis::planner_strategy::types::ReferenceGraph; + +use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; +use crate::bindings::_helpers::model_config::config_values::config_str; +use crate::bindings::_helpers::planner_strategy::microbatches::{ + CAP_ACTIONS, microbatch_limit_config, +}; +use crate::bindings::_helpers::planner_strategy::values::{ + COMPILE_EXCEPTIONS_MODULE, PLANNER_EXCEPTIONS_MODULE, input_error, text, +}; + +const MICROBATCH_MODE: &str = "microbatch"; + +/// The project's models and SQL functions by name. +struct ProjectLineage<'py> { + models_by_name: Bound<'py, PyAny>, + functions_by_name: Bound<'py, PyAny>, +} + +/// The references of the value under `name`, when the mapping has it. +fn mapped_references<'py>( + mapping: &Bound<'py, PyAny>, + name: &str, +) -> PyResult>>> { + let value: Bound<'py, PyAny> = mapping.call_method1("get", (name,))?; + if value.is_none() { + return Ok(None); + } + references(&value).map(Some) +} + +fn references<'py>(owner: &Bound<'py, PyAny>) -> PyResult>> { + owner.getattr("references")?.try_iter()?.collect() +} + +/// Whether a model's `incremental_mode` is microbatch. +fn microbatch_flag(model: &Bound<'_, PyAny>) -> PyResult { + let values: Bound<'_, PyAny> = model.getattr("config")?.getattr("values")?; + Ok(match config_str(&values, "incremental_mode")? { + Some(mode) => mode.eq(MICROBATCH_MODE)?, + None => false, + }) +} + +/// A model's watermark input names. +fn watermark_names(model: &Bound<'_, PyAny>) -> PyResult> { + resolve_cursor_input_roles(model)? + .getattr("watermark_inputs")? + .try_iter()? + .map(|name| text(&name?)) + .collect() +} + +impl<'py> ReferenceGraph for ProjectLineage<'py> { + type Reference = Bound<'py, PyAny>; + type Error = PyErr; + + fn identity(&self, reference: &Bound<'py, PyAny>) -> PyResult { + Ok(ReferenceIdentity { + kind: text(&reference.getattr("ref_kind")?)?, + name: text(&reference.getattr("ref_name")?)?, + }) + } + + fn model_references(&self, name: &str) -> PyResult>>> { + mapped_references(&self.models_by_name, name) + } + + fn function_references(&self, name: &str) -> PyResult>>> { + mapped_references(&self.functions_by_name, name) + } + + fn producer_facts(&self, name: &str) -> PyResult { + let producer: Bound<'py, PyAny> = self.models_by_name.get_item(name)?; + let values: Bound<'py, PyAny> = producer.getattr("config")?.getattr("values")?; + let capped: bool = match microbatch_limit_config(&values)?.1 { + Some(action) => CAP_ACTIONS.contains(&text(&action)?.as_str()), + None => false, + }; + Ok(ProducerFacts { + microbatch: microbatch_flag(&producer)?, + capped, + }) + } + + fn watermark_inputs(&self, name: &str) -> PyResult> { + watermark_names(&self.models_by_name.get_item(name)?) + } +} + +/// The Python error of a failed lineage walk. +fn lineage_error(py: Python<'_>, failure: LineageFailure) -> PyErr { + match failure { + LineageFailure::Graph(error) => error, + LineageFailure::NotInLineage { model, input } => input_error( + py, + (PLANNER_EXCEPTIONS_MODULE, "PlannerInputError"), + &LineageFailure::::message(&model, &input), + Some(LineageFailure::::code()), + ), + } +} + +/// The reference one named input resolves to through model and SQL function references. +#[pyfunction] +#[pyo3(signature = (*, model, input_name, models_by_name, functions_by_name))] +fn resolve_lineage_reference<'py>( + model: &Bound<'py, PyAny>, + input_name: &str, + models_by_name: Bound<'py, PyAny>, + functions_by_name: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = model.py(); + let graph = ProjectLineage { + models_by_name, + functions_by_name, + }; + let name: String = text(&model.getattr("name")?)?; + lineage_reference(&graph, (&name, references(model)?), input_name) + .map_err(|failure| lineage_error(py, failure)) +} + +/// Reject capped producers feeding a microbatch model's watermark inputs. +#[pyfunction(name = "validate_capped_watermark_inputs")] +#[pyo3(signature = (*, model, models_by_name, functions_by_name = None))] +fn reject_capped_watermark_inputs<'py>( + model: &Bound<'py, PyAny>, + models_by_name: Bound<'py, PyAny>, + functions_by_name: Option>, +) -> PyResult<()> { + let py: Python<'py> = model.py(); + if !microbatch_flag(model)? { + return Ok(()); + } + let functions_by_name: Bound<'py, PyAny> = match functions_by_name { + Some(functions) if functions.is_truthy()? => functions, + _ => PyDict::new(py).into_any(), + }; + let graph = ProjectLineage { + models_by_name, + functions_by_name, + }; + let name: String = text(&model.getattr("name")?)?; + let inputs: Vec = watermark_names(model)?; + match capped_watermark_input(&graph, (&name, references(model)?), inputs) { + Ok(None) => Ok(()), + Ok(Some(capped)) => Err(input_error( + py, + (COMPILE_EXCEPTIONS_MODULE, "CompileInputError"), + &capped.message(), + None, + )), + Err(failure) => Err(lineage_error(py, failure)), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_lineage_reference, module)?)?; + module.add_function(wrap_pyfunction!(reject_capped_watermark_inputs, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs new file mode 100644 index 0000000000..916e4d3de5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs @@ -0,0 +1,324 @@ +//! Microbatch counts, capped ranges and batch-limit configuration. + +use std::marker::PhantomData; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyString, PyTuple, PyTupleMethods}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_strategy::main::cap_integer_bounds::cap_integer_bounds; +use sqlbuild_analysis::planner_strategy::main::cap_timeline_bounds::cap_timeline_bounds; +use sqlbuild_analysis::planner_strategy::main::count_integer_batches::count_integer_batches; +use sqlbuild_analysis::planner_strategy::main::count_timeline_batches::count_timeline_batches; +use sqlbuild_analysis::planner_strategy::main::microbatch_limit_warning::microbatch_limit_warning as limit_warning; +use sqlbuild_analysis::planner_strategy::models::{CapDirection, CappedBound}; +use sqlbuild_analysis::planner_strategy::types::Timeline; +use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; +use sqlbuild_cursor::algebra::models::{DurationDigits, DurationSpan}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::cursor::durations::shifted_moment; +use crate::bindings::_helpers::plan_output::display::aligned_timestamp_bounds; +use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; +use crate::bindings::_helpers::planner_strategy::values::text; + +const CURSOR_MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; +const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +const INTEGER_CURSOR: &str = "integer"; +const TIMESTAMP_CURSOR: &str = "timestamp"; +const MONTHS_PER_YEAR: i64 = 12; +const CAP_FROM_START: &str = "cap_from_start"; +const CAP_FROM_END: &str = "cap_from_end"; +/// The limit actions that cap the planned range. +pub(crate) const CAP_ACTIONS: [&str; 2] = [CAP_FROM_END, CAP_FROM_START]; + +/// A start and end bound. +type BoundPair<'py> = (Bound<'py, PyAny>, Bound<'py, PyAny>); +/// An optional configured value. +type MaybeValue<'py> = Option>; + +/// Python `datetime` values one parsed duration apart. +struct DurationTimeline<'py> { + span: DurationSpan, + marker: PhantomData>, +} + +impl<'py> Timeline for DurationTimeline<'py> { + type Moment = Bound<'py, PyAny>; + type Error = PyErr; + + fn advance(&self, moment: &Bound<'py, PyAny>) -> PyResult> { + shifted_moment(moment, &self.span, 1) + } + + fn precedes(&self, left: &Bound<'py, PyAny>, right: &Bound<'py, PyAny>) -> PyResult { + left.lt(right) + } +} + +/// `Duration.parse(batch_size)` as a span, or `None` when it is not a duration. +fn duration_span(batch_size: &str) -> Option { + let digits: DurationDigits = parse_duration(batch_size)?; + let number = |text: &str| -> Option { + text.bytes().try_fold(0_i64, |value, digit| { + value + .checked_mul(10)? + .checked_add(i64::from(digit.checked_sub(b'0')?)) + }) + }; + Some(DurationSpan { + months: number(&digits.years)? * MONTHS_PER_YEAR + number(&digits.months)?, + days: number(&digits.days)?, + hours: number(&digits.hours)?, + minutes: number(&digits.minutes)?, + seconds: number(&digits.seconds)?, + }) +} + +/// The bounds of a `CursorBounds` when each is an instance of one of the cursor value `classes`. +fn typed_bounds<'py>( + bounds: &Bound<'py, PyAny>, + classes: &[&str], +) -> PyResult>> { + let py: Python<'py> = bounds.py(); + let (start, end) = (bounds.getattr("start")?, bounds.getattr("end")?); + for value in [&start, &end] { + let mut typed: bool = false; + for class in classes { + typed = typed || value.is_instance(&imported(py, CURSOR_MODELS_MODULE, class)?)?; + } + if !typed { + return Ok(None); + } + } + Ok(Some((start, end))) +} + +/// The integer values of integer bounds. +fn integer_bounds(bounds: &Bound<'_, PyAny>) -> PyResult> { + let Some((start, end)) = typed_bounds(bounds, &["IntegerValue"])? else { + return Ok(None); + }; + Ok(Some(( + start.getattr("value")?.extract()?, + end.getattr("value")?.extract()?, + ))) +} + +/// The aligned `datetime` bounds of date or timestamp bounds and the batch timeline. +fn timeline_bounds<'py>( + bounds: &Bound<'py, PyAny>, + batch_size: &str, +) -> PyResult, BoundPair<'py>)>> { + let Some(span) = duration_span(batch_size) else { + return Ok(None); + }; + let Some((start, end)) = typed_bounds(bounds, &["DateValue", "TimestampValue"])? else { + return Ok(None); + }; + let aligned: Bound<'py, PyTuple> = aligned_timestamp_bounds(&start, &end)?; + Ok(Some(( + DurationTimeline { + span, + marker: PhantomData, + }, + (aligned.get_item(0)?, aligned.get_item(1)?), + ))) +} + +/// Batches in one concrete cursor range. +#[pyfunction] +#[pyo3(signature = (*, bounds, batch_size, cursor_type, equal_bounds_are_batch = false))] +fn count_microbatches( + bounds: &Bound<'_, PyAny>, + batch_size: &str, + cursor_type: &str, + equal_bounds_are_batch: bool, +) -> PyResult { + if cursor_type == INTEGER_CURSOR { + return Ok(integer_bounds(bounds)?.map_or(0, |range| { + count_integer_batches(range, batch_size, equal_bounds_are_batch) + })); + } + if cursor_type != TIMESTAMP_CURSOR { + return Ok(0); + } + let Some((timeline, range)) = timeline_bounds(bounds, batch_size)? else { + return Ok(0); + }; + count_timeline_batches(&timeline, range, equal_bounds_are_batch) +} + +/// `CursorBounds(start=..., end=...)`. +fn cursor_bounds<'py>( + start: Bound<'py, PyAny>, + end: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = start.py(); + call_with( + &imported(py, MODELS_MODULE, "CursorBounds")?, + &[("start", start), ("end", end)], + ) +} + +/// `(value=value)` of the cursor values module. +fn cursor_value<'py>( + py: Python<'py>, + class: &str, + value: Bound<'py, PyAny>, +) -> PyResult> { + call_with( + &imported(py, CURSOR_MODELS_MODULE, class)?, + &[("value", value)], + ) +} + +/// One concrete range capped to its earliest or latest batch windows. +#[pyfunction] +#[pyo3(signature = (**options))] +fn cap_microbatch_bounds<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "cap_microbatch_bounds", + options, + ( + &[ + "bounds", + "batch_size", + "cursor_type", + "max_batches", + "action", + ], + &[], + ), + )?; + let bounds: Bound<'py, PyAny> = arguments.value("bounds")?; + let batch_size: String = text(&arguments.value("batch_size")?)?; + let cursor_type: String = text(&arguments.value("cursor_type")?)?; + let max_batches: i128 = arguments.value("max_batches")?.extract()?; + let direction: CapDirection = match text(&arguments.value("action")?)?.as_str() { + CAP_FROM_START => CapDirection::FromStart, + CAP_FROM_END => CapDirection::FromEnd, + _ => return Ok(bounds), + }; + let (start, end) = (bounds.getattr("start")?, bounds.getattr("end")?); + let capped: CappedBound> = if cursor_type == INTEGER_CURSOR { + let Some(range) = integer_bounds(&bounds)? else { + return Ok(bounds); + }; + let Some(capped) = cap_integer_bounds(range, &batch_size, max_batches, direction) else { + return Ok(bounds); + }; + match capped { + CappedBound::End(value) => CappedBound::End(cursor_value( + py, + "IntegerValue", + value.into_pyobject(py)?.into_any(), + )?), + CappedBound::Start(value) => CappedBound::Start(cursor_value( + py, + "IntegerValue", + value.into_pyobject(py)?.into_any(), + )?), + } + } else { + if cursor_type != TIMESTAMP_CURSOR { + return Ok(bounds); + } + let Some((timeline, range)) = timeline_bounds(&bounds, &batch_size)? else { + return Ok(bounds); + }; + match cap_timeline_bounds(&timeline, range, max_batches, direction)? { + CappedBound::End(value) => CappedBound::End(cursor_value(py, "TimestampValue", value)?), + CappedBound::Start(value) => { + CappedBound::Start(cursor_value(py, "TimestampValue", value)?) + } + } + }; + match capped { + CappedBound::End(capped_end) => cursor_bounds(start, capped_end), + CappedBound::Start(capped_start) => cursor_bounds(capped_start, end), + } +} + +/// The prominent warning or error message when one model exceeds its batch limit. +#[pyfunction] +#[pyo3(signature = (*, model_name, max_batches, batch_count, action))] +fn microbatch_limit_warning( + model_name: &str, + max_batches: Option, + batch_count: i128, + action: &Bound<'_, PyAny>, +) -> PyResult> { + let action: String = text(action)?; + let cap: bool = CAP_ACTIONS.contains(&action.as_str()); + Ok(limit_warning( + model_name, + (max_batches, batch_count), + (&action, cap), + )) +} + +/// A positive non-boolean integer, or `None`. +fn positive_integer<'py>(value: Option>) -> PyResult>> { + let Some(value) = value else { + return Ok(None); + }; + if value.is_instance_of::() && !value.is_instance_of::() && value.gt(0)? { + return Ok(Some(value)); + } + Ok(None) +} + +/// The configured batch limit and explicit action of validated model config values. +pub(crate) fn microbatch_limit_config<'py>( + values: &Bound<'py, PyAny>, +) -> PyResult<(MaybeValue<'py>, MaybeValue<'py>)> { + let py: Python<'py> = values.py(); + let raw: Bound<'py, PyAny> = values.call_method1("get", ("microbatch_limit",))?; + if let Ok(raw) = raw.downcast::() { + let max_batches: Option> = + positive_integer(raw.get_item("max_batches")?)?; + let action: Option> = raw + .get_item("action")? + .filter(|action| action.is_instance_of::()); + if let (Some(max_batches), Some(action)) = (max_batches, action) { + let action: Bound<'py, PyAny> = + imported(py, CONTRACT_TYPES_MODULE, "MicrobatchLimitAction")?.call1((action,))?; + return Ok((Some(max_batches), Some(action))); + } + } + let legacy: Bound<'py, PyAny> = values.call_method1("get", ("max_microbatches",))?; + Ok((positive_integer(Some(legacy))?, None)) +} + +/// `(max_batches, action)` from validated model config values. +#[pyfunction] +#[pyo3(signature = (*, values))] +fn resolve_microbatch_limit_config<'py>( + values: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = values.py(); + let (max_batches, action) = microbatch_limit_config(values)?; + PyTuple::new( + py, + [ + max_batches.unwrap_or_else(|| py.None().into_bound(py)), + action.unwrap_or_else(|| py.None().into_bound(py)), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(count_microbatches, module)?)?; + module.add_function(wrap_pyfunction!(cap_microbatch_bounds, module)?)?; + module.add_function(wrap_pyfunction!(microbatch_limit_warning, module)?)?; + module.add_function(wrap_pyfunction!(resolve_microbatch_limit_config, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs new file mode 100644 index 0000000000..0ec94542c6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod audits; +pub(crate) mod cursor_types; +pub(crate) mod lineage; +pub(crate) mod microbatches; +pub(crate) mod registration; +pub(crate) mod strategy; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs new file mode 100644 index 0000000000..e4a631cc0f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -0,0 +1,15 @@ +//! Register the planner strategy. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::planner_strategy::{ + audits, cursor_types, lineage, microbatches, strategy, +}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + audits::register(module)?; + cursor_types::register(module)?; + lineage::register(module)?; + microbatches::register(module)?; + strategy::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs new file mode 100644 index 0000000000..26418b9f29 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs @@ -0,0 +1,185 @@ +//! Per-model plan actions, schema actions and plan warnings. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_strategy::main::materialization_type::materialization_type; +use sqlbuild_analysis::planner_strategy::main::model_plan_action::model_plan_action; +use sqlbuild_analysis::planner_strategy::main::model_warnings::model_warnings; +use sqlbuild_analysis::planner_strategy::main::schema_actions::schema_actions; +use sqlbuild_analysis::planner_strategy::models::{ + ModelStrategy, PlanAction, PlanReason, SchemaActionKind, WarningInputs, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_changes::results::{MODELS_MODULE, planner_enum}; +use crate::bindings::_helpers::planner_strategy::values::{ + PLANNER_EXCEPTIONS_MODULE, change_facts, config_value, finding_rows, input_error, + materialization_value, model_materialization, optional_string, plan_warning, text, +}; + +fn action_value(action: PlanAction) -> &'static str { + match action { + PlanAction::CreateView => "create_view", + PlanAction::CreateTable => "create_table", + PlanAction::IncrementalAppend => "incremental_append", + PlanAction::IncrementalDeleteInsert => "incremental_delete_insert", + PlanAction::IncrementalMerge => "incremental_merge", + PlanAction::Snapshot => "snapshot", + PlanAction::Skip => "skip", + PlanAction::Custom => "custom", + } +} + +fn reason_value(reason: PlanReason) -> &'static str { + match reason { + PlanReason::FirstRun => "first_run", + PlanReason::Renamed => "renamed", + PlanReason::FullRefresh => "full_refresh", + PlanReason::QueryChanged => "query_changed", + PlanReason::FunctionChanged => "function_changed", + PlanReason::ConfigChanged => "config_changed", + PlanReason::SchemaChanged => "schema_changed", + PlanReason::NormalIncremental => "normal_incremental", + PlanReason::NoChange => "no_change", + PlanReason::Disabled => "disabled", + } +} + +/// The optional `on_schema_change` policy as text. +fn schema_policy(on_schema_change: &Bound<'_, PyAny>) -> PyResult> { + if on_schema_change.is_none() { + return Ok(None); + } + text(on_schema_change).map(Some) +} + +/// The `MaterializationType` of a model's `materialized` config: table when unset, custom when unknown. +#[pyfunction(name = "get_materialization_type")] +#[pyo3(signature = (model))] +fn materialization_of<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + planner_enum( + model.py(), + "MaterializationType", + materialization_value(model_materialization(model)?), + ) +} + +/// The plan action and reason for one model. +#[pyfunction] +#[pyo3(signature = (*, model, change_result, full_refresh))] +fn resolve_model_plan_action<'py>( + model: &Bound<'py, PyAny>, + change_result: &Bound<'py, PyAny>, + full_refresh: bool, +) -> PyResult> { + let py: Python<'py> = model.py(); + let enabled: Bound<'py, PyAny> = config_value(model, "enabled")?; + let strategy = ModelStrategy { + name: text(&model.getattr("name")?)?, + disabled: enabled.is_instance_of::() && !enabled.is_truthy()?, + materialization: model_materialization(model)?, + incremental_strategy: optional_string(&config_value(model, "incremental_strategy")?)?, + }; + let (action, reason) = + model_plan_action(&strategy, &change_facts(change_result)?, full_refresh).map_err( + |error| { + input_error( + py, + (PLANNER_EXCEPTIONS_MODULE, "PlannerInputError"), + &error.message(), + Some(error.code()), + ) + }, + )?; + PyTuple::new( + py, + [ + planner_enum(py, "PlanAction", action_value(action))?, + planner_enum(py, "PlanReason", reason_value(reason))?, + ], + ) +} + +/// Concrete schema change actions from findings and `on_schema_change`. +#[pyfunction] +#[pyo3(signature = (*, schema_findings, on_schema_change))] +fn resolve_schema_actions<'py>( + schema_findings: &Bound<'py, PyAny>, + on_schema_change: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = schema_findings.py(); + let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "SchemaAction")?; + let policy: Option = schema_policy(on_schema_change)?; + let mut actions: Vec> = Vec::new(); + for row in schema_actions(&finding_rows(schema_findings)?, policy.as_deref()) { + let kind: &str = match row.kind { + SchemaActionKind::AddColumn => "add_column", + SchemaActionKind::DropColumn => "drop_column", + SchemaActionKind::AlterColumnType => "alter_column_type", + }; + actions.push(call_with( + &class, + &[ + ("kind", planner_enum(py, "SchemaActionKind", kind)?), + ( + "column_name", + PyString::new(py, &row.column_name).into_any(), + ), + ("column_type", row.column_type.into_pyobject(py)?.into_any()), + ], + )?); + } + PyTuple::new(py, actions) +} + +/// Warnings for one model from its change detection and plan decisions. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_model_warnings<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_model_warnings", + options, + ( + &[ + "model_name", + "materialization_type", + "change_result", + "schema_actions", + "on_schema_change", + "type_enforcement", + ], + &[], + ), + )?; + let model_name: Bound<'py, PyAny> = arguments.value("model_name")?; + let inputs = WarningInputs { + model_name: text(&model_name)?, + materialization: materialization_type(Some(&text( + &arguments.value("materialization_type")?, + )?)), + has_schema_actions: arguments.flag("schema_actions")?, + on_schema_change: schema_policy(&arguments.value("on_schema_change")?)?, + type_enforcement: arguments.flag("type_enforcement")?, + }; + let rows = model_warnings(&inputs, &change_facts(&arguments.value("change_result")?)?); + let warnings: Vec> = rows + .into_iter() + .map(|row| plan_warning(&model_name, row)) + .collect::>()?; + PyTuple::new(py, warnings) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(materialization_of, module)?)?; + module.add_function(wrap_pyfunction!(resolve_model_plan_action, module)?)?; + module.add_function(wrap_pyfunction!(resolve_schema_actions, module)?)?; + module.add_function(wrap_pyfunction!(build_model_warnings, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs new file mode 100644 index 0000000000..5101af6fd6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs @@ -0,0 +1,168 @@ +//! Conversions between the planner's Python models and the native strategy rows. + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; + +use sqlbuild_analysis::planner_changes::models::{ + SchemaChangeKind, SchemaColumnSource, SchemaFindingRow, +}; +use sqlbuild_analysis::planner_strategy::main::materialization_type::materialization_type; +use sqlbuild_analysis::planner_strategy::models::{ + ChangeFacts, MaterializationType, PlanChangeKind, WarningRow, WarningSeverity, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_changes::results::{MODELS_MODULE, planner_enum}; + +pub(crate) const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +pub(crate) const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const YML_SOURCE: &str = "yml"; +const FULL_BACKFILL: &str = "full"; +const FORWARD_ONLY_BACKFILL: &str = "forward"; + +/// `str(value)` of a Python string or string enum. +pub(crate) fn text(value: &Bound<'_, PyAny>) -> PyResult { + value.extract() +} + +/// A string value, or `None` for `None` and anything that is not a string. +pub(crate) fn optional_string(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_instance_of::() { + return text(value).map(Some); + } + Ok(None) +} + +/// `values.get(key)` of a model's config. +pub(crate) fn config_value<'py>( + model: &Bound<'py, PyAny>, + key: &str, +) -> PyResult> { + model + .getattr("config")? + .getattr("values")? + .call_method1("get", (key,)) +} + +/// The materialization type of a model's `materialized` config. +pub(crate) fn model_materialization(model: &Bound<'_, PyAny>) -> PyResult { + let materialized: Option = optional_string(&config_value(model, "materialized")?)?; + Ok(materialization_type(materialized.as_deref())) +} + +/// The `MaterializationType` value of a materialization. +pub(crate) fn materialization_value(materialization: MaterializationType) -> &'static str { + match materialization { + MaterializationType::View => "view", + MaterializationType::Table => "table", + MaterializationType::Incremental => "incremental", + MaterializationType::Snapshot => "snapshot", + MaterializationType::Seed => "seed", + MaterializationType::Custom => "custom", + } +} + +fn change_kind(value: &str) -> PlanChangeKind { + match value { + "first_run" => PlanChangeKind::FirstRun, + "renamed" => PlanChangeKind::Renamed, + "query_changed" => PlanChangeKind::QueryChanged, + "function_changed" => PlanChangeKind::FunctionChanged, + "config_changed" => PlanChangeKind::ConfigChanged, + "schema_changed" => PlanChangeKind::SchemaChanged, + _ => PlanChangeKind::NoChange, + } +} + +/// One `SchemaFinding` as a native row. +fn finding_row(finding: &Bound<'_, PyAny>) -> PyResult { + let kind: SchemaChangeKind = match text(&finding.getattr("kind")?)?.as_str() { + "column_added" => SchemaChangeKind::ColumnAdded, + "column_removed" => SchemaChangeKind::ColumnRemoved, + _ => SchemaChangeKind::ColumnTypeChanged, + }; + let source: SchemaColumnSource = if text(&finding.getattr("source")?)? == YML_SOURCE { + SchemaColumnSource::Yml + } else { + SchemaColumnSource::SqlAnalysis + }; + Ok(SchemaFindingRow { + kind, + column_name: text(&finding.getattr("column_name")?)?, + source, + expected_type: finding.getattr("expected_type")?.extract()?, + actual_type: finding.getattr("actual_type")?.extract()?, + }) +} + +/// A tuple of `SchemaFinding` values as native rows. +pub(crate) fn finding_rows(findings: &Bound<'_, PyAny>) -> PyResult> { + findings + .try_iter()? + .map(|finding| finding_row(&finding?)) + .collect() +} + +/// The parts of a `ChangeDetectionResult` the strategy reads. +pub(crate) fn change_facts(change: &Bound<'_, PyAny>) -> PyResult { + let backfill: String = text(&change.getattr("backfill")?.getattr("action")?)?; + Ok(ChangeFacts { + kind: change_kind(&text(&change.getattr("change_kind")?)?), + backfill_full: backfill == FULL_BACKFILL, + backfill_forward_only: backfill == FORWARD_ONLY_BACKFILL, + query_changed: change.getattr("query_changed")?.is_truthy()?, + changed_functions: change + .getattr("changed_functions")? + .try_iter()? + .map(|name| text(&name?)) + .collect::>()?, + config_changed: change.getattr("config_changed")?.is_truthy()?, + schema_findings: finding_rows(&change.getattr("schema_findings")?)?, + recorded_build_relation_missing: change + .getattr("recorded_build_relation_missing")? + .is_truthy()?, + }) +} + +/// `PlanWarning(model_name=..., severity=..., message=..., code=...)`. +pub(crate) fn plan_warning<'py>( + model_name: &Bound<'py, PyAny>, + row: WarningRow, +) -> PyResult> { + let py: Python<'py> = model_name.py(); + let severity: &str = match row.severity { + WarningSeverity::Error => "error", + WarningSeverity::Warning => "warning", + WarningSeverity::Info => "info", + }; + call_with( + &imported(py, MODELS_MODULE, "PlanWarning")?, + &[ + ("model_name", model_name.clone()), + ("severity", planner_enum(py, "WarningSeverity", severity)?), + ("message", PyString::new(py, &row.message).into_any()), + ("code", row.code.into_pyobject(py)?.into_any()), + ], + ) +} + +/// `.(message, code=code)` raised as an error. +pub(crate) fn input_error( + py: Python<'_>, + (module, class): (&str, &str), + message: &str, + code: Option<&str>, +) -> PyErr { + let raised: PyResult> = imported(py, module, class).and_then(|class| { + let options: Bound<'_, PyDict> = PyDict::new(py); + if let Some(code) = code { + options.set_item("code", code)?; + } + class.call((message,), Some(&options)) + }); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 167ebb5f47..070cb9fca0 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -7,9 +7,9 @@ use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, contracts, cursor, dag_artifact, diagnostics, discovery, fingerprints, frontier, graph, - lineage, macro_bridge, manifest, model_config, plan_output, planner_changes, project_assembly, - refactor, references, render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, - sqltext, state_rows, type_system, + lineage, macro_bridge, manifest, model_config, plan_output, planner_changes, planner_strategy, + project_assembly, refactor, references, render, rules, scopes, semantic_checks, + semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, }; use crate::bindings::models; @@ -46,6 +46,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { refactor::registration::register(module)?; plan_output::registration::register(module)?; planner_changes::registration::register(module)?; + planner_strategy::registration::register(module)?; references::registration::register(module)?; authored_values::registration::register(module)?; auditing::registration::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8eeeb6e185..7560db38f3 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -62,7 +62,13 @@ from sqlbuild.compiler.auditing.models import ( MeasurementContract, MeasurementThresholds, ) -from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditOutcome, AuditSeverity +from sqlbuild.compiler.auditing.types import ( + AuditAttachmentKind, + AuditEvaluationMode, + AuditOutcome, + AuditRunScope, + AuditSeverity, +) from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.models import ( CompileAdapterContext, @@ -197,12 +203,21 @@ from sqlbuild.compiler.planner.models import ( PlannerRuntime, PlannerScope, PlanOutput, + PlanWarning, ReferenceRename, + SchemaAction, SchemaFinding, SqlTestPlanEntry, WarehouseSnapshot, ) -from sqlbuild.compiler.planner.types import CursorGrain, MaterializationType, PlanAction +from sqlbuild.compiler.planner.types import ( + CursorGrain, + CursorType, + MaterializationType, + OnSchemaChange, + PlanAction, + PlanReason, +) from sqlbuild.compiler.project_assembly.models import NativeProjectResources from sqlbuild.compiler.python_nodes.models import ( PythonNodeGraph, @@ -332,6 +347,7 @@ from sqlbuild.spec.contracts.models import ( TargetConfig, ) from sqlbuild.spec.contracts.types import ( + MicrobatchLimitAction, SourceFreshnessStrategy, SourceFreshnessValueKind, WarehouseGroup, @@ -3846,6 +3862,78 @@ def replaces_incremental_relation( *, materialization_type: MaterializationType, action: PlanAction ) -> bool: ... def version_identity_metadata_payload(metadata_json: str | None) -> Any: ... +def get_materialization_type(model: CompiledModelView) -> MaterializationType: ... +def resolve_model_plan_action( + *, model: CompiledModelView, change_result: ChangeDetectionResult, full_refresh: bool +) -> tuple[PlanAction, PlanReason]: ... +def resolve_schema_actions( + *, schema_findings: tuple[SchemaFinding, ...], on_schema_change: OnSchemaChange | None +) -> tuple[SchemaAction, ...]: ... +def build_model_warnings( + *, + model_name: str, + materialization_type: MaterializationType, + change_result: ChangeDetectionResult, + schema_actions: tuple[SchemaAction, ...], + on_schema_change: OnSchemaChange | None, + type_enforcement: bool, +) -> tuple[PlanWarning, ...]: ... +def resolve_attachment_kind( + *, + audit: CompiledAuditView, + downstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], +) -> tuple[AuditAttachmentKind, str | None]: ... +def reads_outside_target_lineage( + *, + audit: CompiledAuditView, + upstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], +) -> bool: ... +def resolve_effective_run_scope( + *, requested_run_scope: AuditRunScope, attached_model_materialization: str | None +) -> AuditRunScope: ... +def classify_cursor_sql_type(sql_type: str) -> CursorType | None: ... +def check_cursor_type_consistency( + *, + model_name: str, + cursor_column: str | None, + cursor_type: str | None, + warehouse_columns: tuple[ColumnInfo, ...], + sql_analysis_enabled: bool, +) -> PlanWarning | None: ... +def count_microbatches( + *, + bounds: CursorBounds, + batch_size: str, + cursor_type: str, + equal_bounds_are_batch: bool = False, +) -> int: ... +def cap_microbatch_bounds( + *, + bounds: CursorBounds, + batch_size: str, + cursor_type: str, + max_batches: int, + action: MicrobatchLimitAction, +) -> CursorBounds: ... +def microbatch_limit_warning( + *, model_name: str, max_batches: int | None, batch_count: int, action: MicrobatchLimitAction +) -> str | None: ... +def resolve_microbatch_limit_config( + *, values: Mapping[str, object] +) -> tuple[int | None, MicrobatchLimitAction | None]: ... +def resolve_lineage_reference( + *, + model: CompiledModelView, + input_name: str, + models_by_name: Mapping[str, CompiledModelView], + functions_by_name: Mapping[str, CompiledFunctionView], +) -> CompileSqlReference: ... +def validate_capped_watermark_inputs( + *, + model: CompiledModelView, + models_by_name: Mapping[str, CompiledModelView], + functions_by_name: Mapping[str, CompiledFunctionView] | None = None, +) -> None: ... def detect_schema_changes( *, yml_columns: tuple[ColumnInfo, ...], diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py index a325947579..fa198254ff 100644 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py +++ b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import get_materialization_type from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView @@ -20,7 +21,6 @@ plan_function, sql_test_is_selected, ) -from sqlbuild.compiler.planner._helpers.output.strategy import get_materialization_type from sqlbuild.compiler.planner._helpers.planning.scopes import resolve_planner_scopes from sqlbuild.compiler.planner.models import ( AuditPlanEntry, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py index dd78a3f630..93ec534f32 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py @@ -2,7 +2,13 @@ from __future__ import annotations -from sqlbuild._native import assert_no_unresolved_sql_markers, render_audit_sql +from sqlbuild._native import ( + assert_no_unresolved_sql_markers, + reads_outside_target_lineage, + render_audit_sql, + resolve_attachment_kind, + resolve_effective_run_scope, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, @@ -14,11 +20,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledAuditView -from sqlbuild.compiler.planner._helpers.output.audit_scheduling import ( - reads_outside_target_lineage, - resolve_attachment_kind, - resolve_effective_run_scope, -) from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.spec.contracts.models import SourceEntry diff --git a/src/sqlbuild/compiler/planner/_helpers/output/audit_scheduling.py b/src/sqlbuild/compiler/planner/_helpers/output/audit_scheduling.py deleted file mode 100644 index e4851055be..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/audit_scheduling.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Audit attachment resolution and scheduling validation.""" - -from __future__ import annotations - -from sqlbuild.compiler.auditing.types import ( - AuditAttachmentKind, - AuditRunScope, -) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, -) -from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, - CompiledAuditView, - CompiledResourceType, -) -from sqlbuild.compiler.planner._helpers.graph.core import expand_downstream, expand_upstream -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.types import MaterializationType -from sqlbuild.compiler.references.types import SqlReferenceKind - -_ATTACHMENT_KINDS: dict[AttachedAuditTargetKind, AuditAttachmentKind] = { - AttachedAuditTargetKind.MODEL: AuditAttachmentKind.MODEL, - AttachedAuditTargetKind.SOURCE: AuditAttachmentKind.SOURCE, - AttachedAuditTargetKind.SEED: AuditAttachmentKind.SEED, -} - - -def resolve_attachment_kind( - *, - audit: CompiledAuditView, - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> tuple[AuditAttachmentKind, str | None]: - """Resolve audit attachment kind and attached target name; attached audits gate their target.""" - - if audit.attached_target_kind is not None: - if audit.attached_target_name is None: - raise PlannerInputError( - f"audit '{audit.name}': attached audit is missing an attached target name" - ) - return _ATTACHMENT_KINDS[AttachedAuditTargetKind(audit.attached_target_kind)], ( - audit.attached_target_name - ) - return _infer_singular_attachment(audit=audit, downstream_deps=downstream_deps) - - -def reads_outside_target_lineage( - *, - audit: CompiledAuditView, - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> bool: - """Return whether an attached model audit reads anything besides its target and upstream.""" - - if audit.attached_target_kind is None or audit.attached_target_name is None: - return False - target: CompiledObjectKey = CompiledObjectKey( - resource_type=AttachedAuditTargetKind(audit.attached_target_kind).resource_type, - name=audit.attached_target_name, - ) - lineage: frozenset[CompiledObjectKey] = frozenset( - {target, *expand_upstream(key=target, upstream=upstream_deps)} - ) - return any(dep_key not in lineage for dep_key in audit.scope_deps) - - -def resolve_effective_run_scope( - *, - requested_run_scope: AuditRunScope, - attached_model_materialization: str | None, -) -> AuditRunScope: - """Resolve effective run scope after degradation rules.""" - - if requested_run_scope == AuditRunScope.FINAL: - return AuditRunScope.FINAL - - if attached_model_materialization not in ( - MaterializationType.INCREMENTAL, - MaterializationType.SNAPSHOT, - ): - return AuditRunScope.FINAL - - return AuditRunScope.DELTA_AND_FINAL - - -def _infer_singular_attachment( - *, - audit: CompiledAuditView, - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> tuple[AuditAttachmentKind, str | None]: - """Infer attachment for a singular audit from its refs and graph structure.""" - - model_ref_names: list[str] = [ - ref.ref_name for ref in audit.references if ref.ref_kind == SqlReferenceKind.REF - ] - source_ref_names: list[str] = [ - ref.ref_name for ref in audit.references if ref.ref_kind == SqlReferenceKind.SOURCE - ] - - if not model_ref_names and source_ref_names: - attached_source: str = sorted(source_ref_names)[0] - return AuditAttachmentKind.SOURCE, attached_source - - if not model_ref_names and not source_ref_names: - return AuditAttachmentKind.END, None - - if len(model_ref_names) == 1: - return AuditAttachmentKind.MODEL, model_ref_names[0] - - latest_owner: str | None = _find_single_safe_latest_owner( - model_ref_names=model_ref_names, - downstream_deps=downstream_deps, - ) - if latest_owner is not None: - return AuditAttachmentKind.MODEL, latest_owner - - return AuditAttachmentKind.END, None - - -def _find_single_safe_latest_owner( - *, - model_ref_names: list[str], - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> str | None: - """Find the single model that all other referenced models can reach transitively.""" - - model_keys: list[CompiledObjectKey] = [ - CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=name) - for name in model_ref_names - ] - model_key_set: frozenset[CompiledObjectKey] = frozenset(model_keys) - - candidates: list[CompiledObjectKey] = [] - candidate: CompiledObjectKey - for candidate in model_keys: - all_reach: bool = True - other: CompiledObjectKey - for other in model_keys: - if other == candidate: - continue - reachable: frozenset[CompiledObjectKey] = expand_downstream( - key=other, downstream=downstream_deps - ) - if candidate not in reachable: - all_reach = False - break - if all_reach: - candidates.append(candidate) - - if len(candidates) == 1: - return candidates[0].name - - if len(candidates) > 1: - ordered: list[CompiledObjectKey] = sorted(candidates, key=lambda k: k.name) - deepest: CompiledObjectKey = ordered[0] - other_candidate: CompiledObjectKey - for other_candidate in ordered[1:]: - reachable_from_deepest: frozenset[CompiledObjectKey] = expand_downstream( - key=deepest, downstream=downstream_deps - ) - if other_candidate in reachable_from_deepest: - deepest = other_candidate - all_others_reach: bool = True - check_key: CompiledObjectKey - for check_key in model_key_set: - if check_key == deepest: - continue - if deepest not in expand_downstream(key=check_key, downstream=downstream_deps): - all_others_reach = False - break - if all_others_reach: - return deepest.name - - return None diff --git a/src/sqlbuild/compiler/planner/_helpers/output/cursor_type_check.py b/src/sqlbuild/compiler/planner/_helpers/output/cursor_type_check.py deleted file mode 100644 index 419b9b4505..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/cursor_type_check.py +++ /dev/null @@ -1,219 +0,0 @@ -"""Warehouse cursor column type consistency checking.""" - -from __future__ import annotations - -import logging -import re -from typing import Any - -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.planner.constants import POLYGLOT_CUSTOM_DATA_TYPE_NAME -from sqlbuild.compiler.planner.models import PlanWarning -from sqlbuild.compiler.planner.types import CursorType, WarningSeverity -from sqlbuild.compiler.sql_analysis.main.import_polyglot import import_polyglot -from sqlbuild.diagnostics.main.log_debug_event import log_debug_event - -_DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.planner") - -_POLYGLOT_TIMESTAMP_TYPE_NAMES: frozenset[str] = frozenset( - { - "TIMESTAMP", - "TIMESTAMPNTZ", - "TIMESTAMPLTZ", - "TIMESTAMPTZ", - "TIMESTAMP_S", - "TIMESTAMP_MS", - "TIMESTAMP_NS", - "DATE", - "DATE32", - "DATETIME", - "DATETIME2", - "DATETIME64", - "SMALLDATETIME", - } -) - -_POLYGLOT_INTEGER_TYPE_NAMES: frozenset[str] = frozenset( - { - "INT", - "BIG_INT", - "BIGINT", - "SMALL_INT", - "SMALLINT", - "TINY_INT", - "TINYINT", - "MEDIUMINT", - "INT128", - "INT256", - "UINT", - "UBIGINT", - "USMALLINT", - "UTINYINT", - "UMEDIUMINT", - "UINT128", - "UINT256", - } -) -_DECIMAL_INTEGER_TYPE_PATTERN: re.Pattern[str] = re.compile( - r"^(?:DECIMAL|NUMERIC|NUMBER)(?:\(\s*\d+\s*(?:,\s*(\d+)\s*)?\))?$" -) - - -def check_cursor_type_consistency( - *, - model_name: str, - cursor_column: str | None, - cursor_type: str | None, - warehouse_columns: tuple[ColumnInfo, ...], - sql_analysis_enabled: bool, -) -> PlanWarning | None: - """Check whether the warehouse column type is consistent with cursor_type.""" - - if cursor_column is None or cursor_type is None: - return None - - warehouse_type: str | None = _find_column_type( - columns=warehouse_columns, column_name=cursor_column - ) - if warehouse_type is None: - return None - - declared: CursorType | None = _parse_cursor_type(cursor_type) - if declared is None: - return None - - if sql_analysis_enabled: - return _check_with_polyglot( - model_name=model_name, - cursor_column=cursor_column, - declared=declared, - warehouse_type=warehouse_type, - ) - - return _check_with_heuristic( - model_name=model_name, - cursor_column=cursor_column, - declared=declared, - warehouse_type=warehouse_type, - ) - - -def _find_column_type( - *, - columns: tuple[ColumnInfo, ...], - column_name: str, -) -> str | None: - """Find the warehouse type string for a column by name (case-insensitive).""" - - col: ColumnInfo - for col in columns: - if col.name.upper() == column_name.upper(): - return col.type - return None - - -def _parse_cursor_type(cursor_type: str) -> CursorType | None: - """Parse a cursor_type config string to the enum, or None if unrecognized.""" - - try: - return CursorType(cursor_type) - except ValueError: - return None - - -def _check_with_polyglot( - *, - model_name: str, - cursor_column: str, - declared: CursorType, - warehouse_type: str, -) -> PlanWarning | None: - """Classify warehouse type via sql_analysis and return error on clear mismatch.""" - - detected: CursorType | None = classify_cursor_sql_type(warehouse_type) - if detected is None: - return None - - if detected != declared: - return PlanWarning( - model_name=model_name, - severity=WarningSeverity.ERROR, - message=( - f"cursor column '{cursor_column}' has warehouse type '{warehouse_type}' " - f"which is {detected.value}, but cursor_type is '{declared.value}'" - ), - ) - - return None - - -def _check_with_heuristic( - *, - model_name: str, - cursor_column: str, - declared: CursorType, - warehouse_type: str, -) -> PlanWarning | None: - """Classify warehouse type via substring heuristic and warn on mismatch.""" - - detected: CursorType | None = classify_cursor_sql_type(warehouse_type) - if detected is None: - return None - - if detected != declared: - return PlanWarning( - model_name=model_name, - severity=WarningSeverity.WARNING, - message=( - f"cursor column '{cursor_column}' has warehouse type '{warehouse_type}' " - f"which appears to be {detected.value}, but cursor_type is '{declared.value}'" - ), - ) - - return None - - -def classify_cursor_sql_type(sql_type: str) -> CursorType | None: - """Classify a supported cross-adapter SQL cursor type.""" - - normalized: str = " ".join(sql_type.upper().split()) - decimal_match: re.Match[str] | None = _DECIMAL_INTEGER_TYPE_PATTERN.fullmatch(normalized) - if decimal_match is not None: - return CursorType.INTEGER if int(decimal_match.group(1) or 0) == 0 else None - - base_name: str = normalized.partition("(")[0].strip().replace(" ", "_") - if base_name in _POLYGLOT_TIMESTAMP_TYPE_NAMES: - return CursorType.TIMESTAMP - if base_name in _POLYGLOT_INTEGER_TYPE_NAMES: - return CursorType.INTEGER - - return _classify_type_with_polyglot(normalized) - - -def _classify_type_with_polyglot(warehouse_type: str) -> CursorType | None: - """Classify a warehouse type string using sql_analysis type parsing.""" - - polyglot_module: Any = import_polyglot() - - try: - parsed: Any = polyglot_module.parse_data_type(warehouse_type, dialect="generic") - except polyglot_module.PolyglotError as error: - log_debug_event( - logger=_DEBUG_LOGGER, - message="cursor type classification parse failed; falling back", - sqlbuild_warehouse_type=warehouse_type, - sqlbuild_error=str(error), - ) - return None - - args: dict[str, Any] = dict(getattr(parsed, "args", {}) or {}) - type_name: str = str(args.get("data_type", "")).upper() - if type_name == POLYGLOT_CUSTOM_DATA_TYPE_NAME: - type_name = str(args.get("name", "")).upper().replace("_", "") - - if type_name in _POLYGLOT_TIMESTAMP_TYPE_NAMES: - return CursorType.TIMESTAMP - if type_name in _POLYGLOT_INTEGER_TYPE_NAMES: - return CursorType.INTEGER - - return None diff --git a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py b/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py deleted file mode 100644 index 0d162c1326..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Planner-private microbatch range capping and capped-watermark validation.""" - -from __future__ import annotations - -from datetime import datetime - -from sqlbuild._native import aligned_timestamp_bounds, get_config_str, resolve_cursor_input_roles -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompileSqlReference, -) -from sqlbuild.compiler.compile.types import CompiledFunctionView, CompiledModelView -from sqlbuild.compiler.planner._helpers.resolve.lineage import resolve_lineage_reference -from sqlbuild.compiler.planner.main.execution.microbatch_limit import ( - _resolve_microbatch_limit_config, -) -from sqlbuild.compiler.planner.models import CursorBounds, Duration -from sqlbuild.compiler.planner.types import CursorType, IncrementalMode -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue -from sqlbuild.spec.contracts.types import MicrobatchLimitAction - -_CAP_ACTIONS: frozenset[MicrobatchLimitAction] = frozenset( - {MicrobatchLimitAction.CAP_FROM_START, MicrobatchLimitAction.CAP_FROM_END} -) - - -def cap_microbatch_bounds( - *, - bounds: CursorBounds, - batch_size: str, - cursor_type: str, - max_batches: int, - action: MicrobatchLimitAction, -) -> CursorBounds: - """Cap one concrete range to its earliest or latest batch windows.""" - - if action not in { - MicrobatchLimitAction.CAP_FROM_END, - MicrobatchLimitAction.CAP_FROM_START, - }: - return bounds - if cursor_type == CursorType.INTEGER: - if not isinstance(bounds.start, IntegerValue) or not isinstance(bounds.end, IntegerValue): - return bounds - start: int = bounds.start.value - end: int = bounds.end.value - try: - size: int = int(batch_size) - except ValueError: - return bounds - if size <= 0: - return bounds - if action == MicrobatchLimitAction.CAP_FROM_START: - return CursorBounds( - start=bounds.start, - end=IntegerValue(value=min(end, start + size * max_batches)), - ) - batch_count: int = max(0, (end - start + size - 1) // size) - skipped_batches: int = max(0, batch_count - max_batches) - return CursorBounds( - start=IntegerValue(value=start + size * skipped_batches), end=bounds.end - ) - - if cursor_type != CursorType.TIMESTAMP: - return bounds - duration: Duration | None = Duration.parse(batch_size) - if duration is None: - return bounds - if not isinstance(bounds.start, DateValue | TimestampValue) or not isinstance( - bounds.end, DateValue | TimestampValue - ): - return bounds - start_at, end_at = aligned_timestamp_bounds(start=bounds.start, end=bounds.end) - if action == MicrobatchLimitAction.CAP_FROM_START: - capped_end: datetime = start_at - for _ in range(max_batches): - capped_end = min(duration.add_to(capped_end), end_at) - return CursorBounds(start=bounds.start, end=TimestampValue(value=capped_end)) - boundaries: list[datetime] = [start_at] - while boundaries[-1] < end_at: - boundaries.append(min(duration.add_to(boundaries[-1]), end_at)) - capped_start: datetime = boundaries[max(0, len(boundaries) - 1 - max_batches)] - return CursorBounds(start=TimestampValue(value=capped_start), end=bounds.end) - - -def validate_capped_watermark_inputs( - *, - model: CompiledModelView, - models_by_name: dict[str, CompiledModelView], - functions_by_name: dict[str, CompiledFunctionView] | None = None, -) -> None: - """Reject capped producers feeding microbatch watermark inputs.""" - - if ( - get_config_str(values=model.config.values, key="incremental_mode") - != IncrementalMode.MICROBATCH - ): - return - input_name: str - for input_name in resolve_cursor_input_roles(model=model).watermark_inputs: - ref: CompileSqlReference = resolve_lineage_reference( - model=model, - input_name=input_name, - models_by_name=models_by_name, - functions_by_name=functions_by_name or {}, - ) - producer: CompiledModelView | None = _find_capped_producer_ancestor( - reference=ref, - models_by_name=models_by_name, - functions_by_name=functions_by_name or {}, - ) - if producer is None: - continue - if producer.name != input_name: - raise CompileInputError( - f"model '{model.name}' uses watermark input '{input_name}' derived from capped " - f"producer '{producer.name}'; capped producers cannot serve as watermark inputs" - ) - raise CompileInputError( - f"model '{model.name}' uses capped producer '{producer.name}' as a watermark input; " - "capped producers cannot serve as watermark inputs" - ) - - -def _find_capped_producer_ancestor( - *, - reference: CompileSqlReference, - models_by_name: dict[str, CompiledModelView], - functions_by_name: dict[str, CompiledFunctionView], -) -> CompiledModelView | None: - """Return the first capped microbatch model in a reference's upstream lineage.""" - - pending: list[CompileSqlReference] = [reference] - visited: set[tuple[str, str]] = set() - while pending: - current: CompileSqlReference = pending.pop(0) - identity: tuple[str, str] = (current.ref_kind, current.ref_name) - if identity in visited: - continue - visited.add(identity) - if current.ref_kind == SqlReferenceKind.REF: - producer: CompiledModelView | None = models_by_name.get(current.ref_name) - if producer is None: - continue - action: MicrobatchLimitAction | None = _resolve_microbatch_limit_config( - values=producer.config.values - )[1] - is_microbatch: bool = ( - get_config_str(values=producer.config.values, key="incremental_mode") - == IncrementalMode.MICROBATCH - ) - if is_microbatch and action in _CAP_ACTIONS: - return producer - if is_microbatch: - watermark_name: str - for watermark_name in resolve_cursor_input_roles(model=producer).watermark_inputs: - pending.append( - resolve_lineage_reference( - model=producer, - input_name=watermark_name, - models_by_name=models_by_name, - functions_by_name=functions_by_name, - ) - ) - else: - pending.extend(producer.references) - elif current.ref_kind in { - SqlReferenceKind.UDF, - SqlReferenceKind.TABLE_FUNCTION, - }: - function: CompiledFunctionView | None = functions_by_name.get(current.ref_name) - if function is not None: - pending.extend(function.references) - return None diff --git a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_count.py b/src/sqlbuild/compiler/planner/_helpers/output/microbatch_count.py deleted file mode 100644 index 247c33d9a8..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_count.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Planner-private microbatch count calculation.""" - -from __future__ import annotations - -from datetime import datetime - -from sqlbuild._native import aligned_timestamp_bounds -from sqlbuild.compiler.planner.models import CursorBounds, Duration -from sqlbuild.compiler.planner.types import CursorType -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue - - -def count_microbatches( - *, bounds: CursorBounds, batch_size: str, cursor_type: str, equal_bounds_are_batch: bool = False -) -> int: - """Count batches for one concrete cursor range.""" - - if cursor_type == CursorType.INTEGER: - if not isinstance(bounds.start, IntegerValue) or not isinstance(bounds.end, IntegerValue): - return 0 - start: int = bounds.start.value - end: int = bounds.end.value - try: - size: int = int(batch_size) - except ValueError: - return 0 - if equal_bounds_are_batch and start == end: - return 1 - if size <= 0 or start >= end: - return 0 - return (end - start + size - 1) // size - if cursor_type != CursorType.TIMESTAMP: - return 0 - duration: Duration | None = Duration.parse(batch_size) - if duration is None: - return 0 - if not isinstance(bounds.start, DateValue | TimestampValue) or not isinstance( - bounds.end, DateValue | TimestampValue - ): - return 0 - current: datetime - end_at: datetime - current, end_at = aligned_timestamp_bounds(start=bounds.start, end=bounds.end) - if equal_bounds_are_batch and current == end_at: - return 1 - count: int = 0 - while current < end_at: - current = min(duration.add_to(current), end_at) - count += 1 - return count diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index c4bc2a6e0a..7a9d8f542e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -7,24 +7,35 @@ from decimal import Decimal from typing import Any -from sqlbuild._native import cursor_parse as parse -from sqlbuild._native import cursor_render as render -from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild._native import ( + build_model_warnings, + cap_microbatch_bounds, + check_cursor_type_consistency, + count_microbatches, detect_model_changes, get_config_bool, get_config_cursor_bound, get_config_int, get_config_str, get_config_string_tuple, + get_materialization_type, microbatch_guidance, + microbatch_limit_warning, render_source_relation, replaces_incremental_relation, resolve_bounded_cursor_override, resolve_cursor_input_roles, resolve_effective_microbatch_batch_size, + resolve_lineage_reference, + resolve_microbatch_limit_config, + resolve_model_plan_action, + resolve_schema_actions, source_binding_diagnostics, + validate_capped_watermark_inputs, ) +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY @@ -47,20 +58,6 @@ ) from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.graph.source_load_nodes import build_source_load_map -from sqlbuild.compiler.planner._helpers.output.cursor_type_check import ( - check_cursor_type_consistency, -) -from sqlbuild.compiler.planner._helpers.output.microbatch_bounds import ( - cap_microbatch_bounds, - validate_capped_watermark_inputs, -) -from sqlbuild.compiler.planner._helpers.output.microbatch_count import count_microbatches -from sqlbuild.compiler.planner._helpers.output.strategy import ( - build_model_warnings, - get_materialization_type, - resolve_model_plan_action, - resolve_schema_actions, -) from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( check_full_rebuild_allowed, describe_full_rebuild_cause, @@ -77,7 +74,6 @@ resolve_future_cursor_config, resolve_start_cursor_config, ) -from sqlbuild.compiler.planner._helpers.resolve.lineage import resolve_lineage_reference from sqlbuild.compiler.planner._helpers.resolve.refs import ( apply_deferred_locations, build_function_locations, @@ -101,10 +97,6 @@ from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety from sqlbuild.compiler.planner.main.execution.future_cursor_warning import future_cursor_cap_warning from sqlbuild.compiler.planner.main.execution.maximum_start_warning import maximum_start_cap_warning -from sqlbuild.compiler.planner.main.execution.microbatch_limit import ( - _resolve_microbatch_limit_config, - microbatch_limit_warning, -) from sqlbuild.compiler.planner.models import ( BackfillResult, ChangeDetectionResult, @@ -928,8 +920,8 @@ def plan_model_from_change( values=model.config.values, key="unaccounted_partition_policy" ), microbatch_range=microbatch_range, - microbatch_limit=_resolve_microbatch_limit_config(values=model.config.values)[0], - declared_microbatch_limit_action=_resolve_microbatch_limit_config( + microbatch_limit=resolve_microbatch_limit_config(values=model.config.values)[0], + declared_microbatch_limit_action=resolve_microbatch_limit_config( values=model.config.values )[1], start_cursor_override=start_cursor_override, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py deleted file mode 100644 index 59f9967230..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py +++ /dev/null @@ -1,389 +0,0 @@ -"""Per-model plan action resolution, schema action resolution, and logical DDL generation.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.discovery.constants import ( - PROJECT_CONFIG_FILENAME, - QUERY_CHANGE_TRACKING_SETTING_KEY, - SETTINGS_SECTION, -) -from sqlbuild.compiler.planner.constants import RECORDED_RELATION_MISSING_WARNING_CODE -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - ChangeDetectionResult, - PlanWarning, - SchemaAction, - SchemaFinding, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - ChangeKind, - IncrementalStrategy, - MaterializationType, - OnSchemaChange, - PlanAction, - PlanReason, - SchemaActionKind, - SchemaChangeKind, - SchemaColumnSource, - WarningSeverity, -) -from sqlbuild.errors.setting_help.main.setting_help import setting_help - -_DEFAULT_ON_SCHEMA_CHANGE: OnSchemaChange = OnSchemaChange.APPEND_NEW_COLUMNS -_OWN_CHANGE_REASONS: dict[ChangeKind, PlanReason] = { - ChangeKind.FIRST_RUN: PlanReason.FIRST_RUN, - ChangeKind.QUERY_CHANGED: PlanReason.QUERY_CHANGED, - ChangeKind.FUNCTION_CHANGED: PlanReason.FUNCTION_CHANGED, - ChangeKind.CONFIG_CHANGED: PlanReason.CONFIG_CHANGED, - ChangeKind.SCHEMA_CHANGED: PlanReason.SCHEMA_CHANGED, -} -_VIEW_CHANGE_REASONS: dict[ChangeKind, PlanReason] = { - **_OWN_CHANGE_REASONS, - ChangeKind.RENAMED: PlanReason.RENAMED, -} - - -def resolve_model_plan_action( - *, - model: CompiledModelView, - change_result: ChangeDetectionResult, - full_refresh: bool, -) -> tuple[PlanAction, PlanReason]: - """Determine the plan action and reason for a single model.""" - - if _is_disabled(model): - return PlanAction.SKIP, PlanReason.DISABLED - - materialization: MaterializationType = get_materialization_type(model) - - if materialization == MaterializationType.CUSTOM: - return PlanAction.CUSTOM, _custom_reason( - change_result=change_result, full_refresh=full_refresh - ) - - if materialization == MaterializationType.VIEW: - return PlanAction.CREATE_VIEW, _view_reason( - change_result=change_result, full_refresh=full_refresh - ) - - if materialization == MaterializationType.SNAPSHOT: - return PlanAction.SNAPSHOT, _snapshot_reason( - change_result=change_result, full_refresh=full_refresh - ) - - if full_refresh: - return PlanAction.CREATE_TABLE, PlanReason.FULL_REFRESH - - if change_result.change_kind == ChangeKind.FIRST_RUN: - return PlanAction.CREATE_TABLE, PlanReason.FIRST_RUN - - if change_result.change_kind == ChangeKind.RENAMED: - return PlanAction.CREATE_TABLE, PlanReason.RENAMED - - if change_result.backfill.action == BackfillAction.FULL: - reason: PlanReason = _backfill_reason(change_result) - return PlanAction.CREATE_TABLE, reason - - if materialization == MaterializationType.TABLE: - return _table_action(change_result) - - return _incremental_action(model=model, change_result=change_result) - - -def resolve_schema_actions( - *, - schema_findings: tuple[SchemaFinding, ...], - on_schema_change: OnSchemaChange | None, -) -> tuple[SchemaAction, ...]: - """Resolve concrete schema change actions from findings and on_schema_change config.""" - - effective: OnSchemaChange = on_schema_change or _DEFAULT_ON_SCHEMA_CHANGE - - if effective == OnSchemaChange.IGNORE or effective == OnSchemaChange.FAIL: - return () - - actions: list[SchemaAction] = [] - finding: SchemaFinding - for finding in schema_findings: - if effective == OnSchemaChange.APPEND_NEW_COLUMNS: - if finding.kind == SchemaChangeKind.COLUMN_ADDED: - actions.append( - SchemaAction( - kind=SchemaActionKind.ADD_COLUMN, - column_name=finding.column_name, - column_type=finding.expected_type, - ) - ) - elif effective == OnSchemaChange.SYNC_ALL_COLUMNS: - if finding.kind == SchemaChangeKind.COLUMN_ADDED: - actions.append( - SchemaAction( - kind=SchemaActionKind.ADD_COLUMN, - column_name=finding.column_name, - column_type=finding.expected_type, - ) - ) - elif finding.kind == SchemaChangeKind.COLUMN_REMOVED: - actions.append( - SchemaAction( - kind=SchemaActionKind.DROP_COLUMN, - column_name=finding.column_name, - ) - ) - elif finding.kind == SchemaChangeKind.COLUMN_TYPE_CHANGED: - actions.append( - SchemaAction( - kind=SchemaActionKind.ALTER_COLUMN_TYPE, - column_name=finding.column_name, - column_type=finding.expected_type, - ) - ) - - return tuple(actions) - - -def build_model_warnings( - *, - model_name: str, - materialization_type: MaterializationType, - change_result: ChangeDetectionResult, - schema_actions: tuple[SchemaAction, ...], - on_schema_change: OnSchemaChange | None, - type_enforcement: bool, -) -> tuple[PlanWarning, ...]: - """Build warnings for a single model based on change detection and plan decisions.""" - - effective: OnSchemaChange = on_schema_change or _DEFAULT_ON_SCHEMA_CHANGE - warnings: list[PlanWarning] = [] - - if change_result.recorded_build_relation_missing: - warnings.append( - PlanWarning( - model_name=model_name, - severity=WarningSeverity.WARNING, - code=RECORDED_RELATION_MISSING_WARNING_CODE, - message=( - f"{model_name}: SQLBuild state records a previous build, but the relation " - "no longer exists in the warehouse. It may have been dropped outside " - "SQLBuild; planning a first run to recreate it." - ), - ) - ) - - if effective == OnSchemaChange.FAIL and change_result.schema_findings: - warnings.append( - PlanWarning( - model_name=model_name, - severity=WarningSeverity.ERROR, - message="schema change detected and on_schema_change is set to fail", - ) - ) - - finding: SchemaFinding - for finding in change_result.schema_findings: - if type_enforcement and finding.source == SchemaColumnSource.YML: - if finding.kind == SchemaChangeKind.COLUMN_TYPE_CHANGED: - warnings.append( - PlanWarning( - model_name=model_name, - severity=WarningSeverity.WARNING, - message=( - f"enforced column {finding.column_name} type mismatch: " - f"expected {finding.expected_type}, " - f"warehouse has {finding.actual_type}" - ), - ) - ) - elif finding.source == SchemaColumnSource.SQL_ANALYSIS: - if finding.kind in ( - SchemaChangeKind.COLUMN_ADDED, - SchemaChangeKind.COLUMN_TYPE_CHANGED, - ): - warnings.append( - PlanWarning( - model_name=model_name, - severity=WarningSeverity.INFO, - message=( - f"inferred column {finding.column_name}: {_describe_finding(finding)}" - ), - ) - ) - - if ( - materialization_type == MaterializationType.INCREMENTAL - and (change_result.query_changed or change_result.changed_functions) - and change_result.backfill.action == BackfillAction.FORWARD_ONLY - ): - changed_subject: str = ( - "the model's SQL changed" - if change_result.query_changed - else f"function {', '.join(change_result.changed_functions)} changed" - ) - warnings.append( - PlanWarning( - model_name=model_name, - severity=WarningSeverity.WARNING, - message=( - f"{model_name}: {changed_subject}, so already-built dates keep " - "their old results - only new dates will use the new query. To apply " - "the change to existing data, choose how much to reprocess: " - "replay_on_change full (rebuild every date with the new SQL), " - "replay_on_change bounded- e.g. bounded-14d (reprocess just " - "that window), or leave it forward-only (existing dates as-is, new " - "dates use the new query - the default). If this model's output does " - "not depend on the SQL that changed, this is safe to ignore. " - + setting_help( - purpose="To silence these warnings", - file_name=PROJECT_CONFIG_FILENAME, - section=SETTINGS_SECTION, - key=QUERY_CHANGE_TRACKING_SETTING_KEY, - value=False, - ) - ), - ) - ) - - has_schema_backfill_findings: bool = any( - f.kind in (SchemaChangeKind.COLUMN_ADDED, SchemaChangeKind.COLUMN_TYPE_CHANGED) - for f in change_result.schema_findings - ) - if has_schema_backfill_findings and not schema_actions: - if effective == OnSchemaChange.IGNORE: - warnings.append( - PlanWarning( - model_name=model_name, - severity=WarningSeverity.INFO, - message="schema change detected but on_schema_change is set to ignore", - ) - ) - - return tuple(warnings) - - -def _is_disabled(model: CompiledModelView) -> bool: - """Check whether a model is explicitly disabled.""" - - raw: object | None = model.config.values.get("enabled") - if isinstance(raw, bool): - return not raw - return False - - -def _custom_reason(*, change_result: ChangeDetectionResult, full_refresh: bool) -> PlanReason: - """Determine the reason for a custom materialization action.""" - - if full_refresh: - return PlanReason.FULL_REFRESH - return _OWN_CHANGE_REASONS.get(change_result.change_kind, PlanReason.NO_CHANGE) - - -def _view_reason(*, change_result: ChangeDetectionResult, full_refresh: bool) -> PlanReason: - """Determine the reason for a view action.""" - - if full_refresh: - return PlanReason.FULL_REFRESH - return _VIEW_CHANGE_REASONS.get(change_result.change_kind, PlanReason.NO_CHANGE) - - -def _snapshot_reason(*, change_result: ChangeDetectionResult, full_refresh: bool) -> PlanReason: - """Determine the reason for a snapshot materialization action.""" - - if full_refresh: - return PlanReason.FULL_REFRESH - return _OWN_CHANGE_REASONS.get(change_result.change_kind, PlanReason.NORMAL_INCREMENTAL) - - -def _backfill_reason(change_result: ChangeDetectionResult) -> PlanReason: - """Determine the reason when backfill forces a full rebuild.""" - - if change_result.query_changed: - return PlanReason.QUERY_CHANGED - if change_result.changed_functions: - return PlanReason.FUNCTION_CHANGED - if change_result.config_changed: - return PlanReason.CONFIG_CHANGED - if change_result.schema_findings: - return PlanReason.SCHEMA_CHANGED - return PlanReason.FULL_REFRESH - - -def _table_action( - change_result: ChangeDetectionResult, -) -> tuple[PlanAction, PlanReason]: - """Determine action for a table materialization (non-incremental).""" - - if change_result.change_kind == ChangeKind.QUERY_CHANGED: - return PlanAction.CREATE_TABLE, PlanReason.QUERY_CHANGED - if change_result.change_kind == ChangeKind.FUNCTION_CHANGED: - return PlanAction.CREATE_TABLE, PlanReason.FUNCTION_CHANGED - if change_result.change_kind == ChangeKind.CONFIG_CHANGED: - return PlanAction.CREATE_TABLE, PlanReason.CONFIG_CHANGED - if change_result.change_kind == ChangeKind.SCHEMA_CHANGED: - return PlanAction.CREATE_TABLE, PlanReason.SCHEMA_CHANGED - return PlanAction.CREATE_TABLE, PlanReason.NO_CHANGE - - -def _incremental_action( - *, - model: CompiledModelView, - change_result: ChangeDetectionResult, -) -> tuple[PlanAction, PlanReason]: - """Determine action for an incremental materialization.""" - - raw_strategy: object | None = model.config.values.get("incremental_strategy") - if not isinstance(raw_strategy, str): - raise PlannerInputError( - f"incremental model '{model.name}' is missing required incremental_strategy", - code="S201", - ) - - reason: PlanReason - if change_result.change_kind == ChangeKind.QUERY_CHANGED: - reason = PlanReason.QUERY_CHANGED - elif change_result.change_kind == ChangeKind.FUNCTION_CHANGED: - reason = PlanReason.FUNCTION_CHANGED - elif change_result.change_kind == ChangeKind.CONFIG_CHANGED: - reason = PlanReason.CONFIG_CHANGED - elif change_result.change_kind == ChangeKind.SCHEMA_CHANGED: - reason = PlanReason.SCHEMA_CHANGED - else: - reason = PlanReason.NORMAL_INCREMENTAL - - action_map: dict[str, PlanAction] = { - IncrementalStrategy.APPEND: PlanAction.INCREMENTAL_APPEND, - IncrementalStrategy.DELETE_INSERT: PlanAction.INCREMENTAL_DELETE_INSERT, - IncrementalStrategy.MERGE: PlanAction.INCREMENTAL_MERGE, - } - action: PlanAction | None = action_map.get(raw_strategy) - if action is None: - raise PlannerInputError( - f"incremental model '{model.name}' has unknown strategy '{raw_strategy}'", - code="S202", - ) - - return action, reason - - -def _describe_finding(finding: SchemaFinding) -> str: - """Build a human-readable description for one schema finding.""" - - if finding.kind == SchemaChangeKind.COLUMN_ADDED: - type_suffix: str = f" ({finding.expected_type})" if finding.expected_type else "" - return f"new column not in warehouse{type_suffix}" - if finding.kind == SchemaChangeKind.COLUMN_TYPE_CHANGED: - return f"expected {finding.expected_type}, warehouse has {finding.actual_type}" - return f"{finding.kind.value}" - - -def get_materialization_type(model: CompiledModelView) -> MaterializationType: - """Extract materialization type from model config.""" - - raw: object | None = model.config.values.get("materialized") - if isinstance(raw, str): - try: - return MaterializationType(raw) - except ValueError: - return MaterializationType.CUSTOM - return MaterializationType.TABLE diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/lineage.py b/src/sqlbuild/compiler/planner/_helpers/resolve/lineage.py deleted file mode 100644 index 1e3929e4d5..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/lineage.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Resolve declared cursor inputs from transitive SQL resource lineage.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import ( - CompileSqlReference, -) -from sqlbuild.compiler.compile.types import CompiledFunctionView, CompiledModelView -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.references.types import SqlReferenceKind - - -def resolve_lineage_reference( - *, - model: CompiledModelView, - input_name: str, - models_by_name: dict[str, CompiledModelView], - functions_by_name: dict[str, CompiledFunctionView], -) -> CompileSqlReference: - """Resolve one named input through model and SQL function references.""" - - pending: list[CompileSqlReference] = list(model.references) - visited: set[tuple[str, str]] = set() - while pending: - reference: CompileSqlReference = pending.pop(0) - if reference.ref_name == input_name: - return reference - identity: tuple[str, str] = (reference.ref_kind, reference.ref_name) - if identity in visited: - continue - visited.add(identity) - if reference.ref_kind == SqlReferenceKind.REF: - upstream_model: CompiledModelView | None = models_by_name.get(reference.ref_name) - if upstream_model is not None: - pending.extend(upstream_model.references) - elif reference.ref_kind in { - SqlReferenceKind.UDF, - SqlReferenceKind.TABLE_FUNCTION, - }: - upstream_function: CompiledFunctionView | None = functions_by_name.get( - reference.ref_name - ) - if upstream_function is not None: - pending.extend(upstream_function.references) - raise PlannerInputError( - f"model '{model.name}': cursor_inputs watermark references '{input_name}', but it is " - "not in the model's upstream lineage", - code="S302", - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py index 02625c3ac9..17d3d343e8 100644 --- a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py +++ b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import get_materialization_type from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -24,7 +25,6 @@ from sqlbuild.compiler.planner._helpers.identity.functions import ( build_compiled_function_fingerprint_sql, ) -from sqlbuild.compiler.planner._helpers.output.strategy import get_materialization_type from sqlbuild.compiler.planner._helpers.resolve.refs import ( build_function_locations, build_model_locations, diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py index 41dd53f435..ccff286819 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py @@ -9,20 +9,22 @@ from datetime import datetime from typing import Any -from sqlbuild._native import cursor_compare as compare -from sqlbuild._native import cursor_exclusive_to_inclusive as exclusive_to_inclusive -from sqlbuild._native import cursor_observed_partition as observed_partition -from sqlbuild._native import cursor_parse as parse -from sqlbuild._native import cursor_render as render from sqlbuild._native import ( + classify_cursor_sql_type, get_config_cursor_bound, get_config_str, read_latest_fingerprints, render_source_relation, resolve_cursor_input_roles, + resolve_lineage_reference, select_latest_fingerprints, transitive_closure_many, ) +from sqlbuild._native import cursor_compare as compare +from sqlbuild._native import cursor_exclusive_to_inclusive as exclusive_to_inclusive +from sqlbuild._native import cursor_observed_partition as observed_partition +from sqlbuild._native import cursor_parse as parse +from sqlbuild._native import cursor_render as render from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapter.contract.types import AdapterExecute @@ -66,7 +68,6 @@ from sqlbuild.compiler.planner._helpers.graph.loader_dag import ( build_upstream_intermediate_source_map, ) -from sqlbuild.compiler.planner._helpers.output.cursor_type_check import classify_cursor_sql_type from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( effectively_full_refreshed_model_names, ) @@ -75,7 +76,6 @@ resolve_effective_timestamp_grain, ) from sqlbuild.compiler.planner._helpers.resolve.cursor_policies import resolve_start_cursor_config -from sqlbuild.compiler.planner._helpers.resolve.lineage import resolve_lineage_reference from sqlbuild.compiler.planner._helpers.resolve.maximum_start import maximum_allowed_start from sqlbuild.compiler.planner.constants import METADATA_NAME_FILTER_LIMIT from sqlbuild.compiler.planner.exceptions import PlannerInputError diff --git a/src/sqlbuild/compiler/planner/main/execution/display_plan.py b/src/sqlbuild/compiler/planner/main/execution/display_plan.py index b962f8575d..549d9a0158 100644 --- a/src/sqlbuild/compiler/planner/main/execution/display_plan.py +++ b/src/sqlbuild/compiler/planner/main/execution/display_plan.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild._native import get_config_str +from sqlbuild._native import get_config_str, get_materialization_type from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) @@ -11,7 +11,6 @@ build_downstream_deps, build_execution_upstream_deps, ) -from sqlbuild.compiler.planner._helpers.output.strategy import get_materialization_type from sqlbuild.compiler.planner._helpers.planning.full_refresh import resolve_model_full_refresh from sqlbuild.compiler.planner.models import BackfillResult, ModelPlanEntry, PlanOutput from sqlbuild.compiler.planner.types import ( diff --git a/src/sqlbuild/compiler/planner/main/execution/microbatch_limit.py b/src/sqlbuild/compiler/planner/main/execution/microbatch_limit.py deleted file mode 100644 index 2d7677f5c7..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/microbatch_limit.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Microbatch planning limit decisions shared by planner and runtime.""" - -from collections.abc import Mapping -from typing import cast - -from sqlbuild.spec.contracts.types import MicrobatchLimitAction - - -def _resolve_microbatch_limit_config( - *, values: Mapping[str, object] -) -> tuple[int | None, MicrobatchLimitAction | None]: - """Return the configured limit and explicit action from validated model values.""" - - raw: object | None = values.get("microbatch_limit") - if isinstance(raw, dict): - raw_config: Mapping[str, object] = cast("Mapping[str, object]", raw) - max_batches: object | None = raw_config.get("max_batches") - action: object | None = raw_config.get("action") - if ( - isinstance(max_batches, int) - and not isinstance(max_batches, bool) - and max_batches > 0 - and isinstance(action, str) - ): - return max_batches, MicrobatchLimitAction(action) - legacy_limit: object | None = values.get("max_microbatches") - return ( - legacy_limit - if isinstance(legacy_limit, int) and not isinstance(legacy_limit, bool) and legacy_limit > 0 - else None, - None, - ) - - -def microbatch_limit_warning( - *, model_name: str, max_batches: int | None, batch_count: int, action: MicrobatchLimitAction -) -> str | None: - """Return the prominent warning/error message when one model exceeds its limit.""" - - if max_batches is None or batch_count <= max_batches: - return None - return ( - f"MICROBATCH LIMIT EXCEEDED: model '{model_name}' planned {batch_count} batches, " - f"above the per-model limit of {max_batches} (action={action.value}). " - + ( - f"Selected {max_batches} batches and deferred {batch_count - max_batches}." - if action - in { - MicrobatchLimitAction.CAP_FROM_END, - MicrobatchLimitAction.CAP_FROM_START, - } - else ( - "Use --max-microbatches with an intentional invocation-specific value for " - "this backfill." - ) - ) - ) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index c5c1454d21..8d0eba4ad8 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -16,6 +16,7 @@ compute_query_hash, cursor_bound_display, inclusive_cursor_end, + microbatch_limit_warning, resolve_effective_microbatch_batch_size, ) from sqlbuild._native import cursor_cap_from_end as cap_from_end @@ -44,9 +45,6 @@ from sqlbuild.compiler.planner.main.cursor_window.empty_cursor_window import empty_cursor_window from sqlbuild.compiler.planner.main.execution.future_cursor_warning import future_cursor_cap_warning from sqlbuild.compiler.planner.main.execution.maximum_start_warning import maximum_start_cap_warning -from sqlbuild.compiler.planner.main.execution.microbatch_limit import ( - microbatch_limit_warning, -) from sqlbuild.compiler.planner.models import ( CursorBounds, Duration, diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py index 30a318d01d..36e7280746 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -116,6 +116,7 @@ def test_given_native_source_when_checking_module_paths_then_reports_errors( "sqlbuild.executor.runtime", "sqlbuild.fingerprints", "sqlbuild.lineage", + "sqlbuild.planner", "sqlbuild.planner.changes", ), ), diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index c07c411f1b..c3c95b77f6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -5,16 +5,18 @@ import pytest -from sqlbuild._native import compute_query_hash, detect_changes, detect_model_changes -from sqlbuild.compiler.compile.models import InferredColumn -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.output.strategy import ( +from sqlbuild._native import ( build_model_warnings, + compute_query_hash, + detect_changes, + detect_model_changes, get_materialization_type, resolve_model_plan_action, ) +from sqlbuild.compiler.compile.models import InferredColumn +from sqlbuild.compiler.compile.types import CompiledModelView +from sqlbuild.compiler.discovery.models import SqlHookEntry +from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.constants import RECORDED_RELATION_MISSING_WARNING_CODE from sqlbuild.compiler.planner.main.identity._version_identity_metadata import ( build_version_identity_metadata_json, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py index cbdff45f0a..4264d405d9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py @@ -4,6 +4,11 @@ import pytest +from sqlbuild._native import ( + reads_outside_target_lineage, + resolve_attachment_kind, + resolve_effective_run_scope, +) from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditRunScope, @@ -16,11 +21,6 @@ AttachedAuditTargetKind, CompiledAuditView, ) -from sqlbuild.compiler.planner._helpers.output.audit_scheduling import ( - reads_outside_target_lineage, - resolve_attachment_kind, - resolve_effective_run_scope, -) from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_type_check.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_type_check.py index a13bec6358..8e9db31606 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_type_check.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_type_check.py @@ -4,10 +4,8 @@ import pytest +from sqlbuild._native import check_cursor_type_consistency from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.planner._helpers.output.cursor_type_check import ( - check_cursor_type_consistency, -) from sqlbuild.compiler.planner.models import PlanWarning from sqlbuild.compiler.planner.types import WarningSeverity from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py index 176bf94ee9..02c08d1056 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py @@ -2,8 +2,8 @@ import pytest +from sqlbuild._native import resolve_model_plan_action from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner._helpers.output.strategy import resolve_model_plan_action from sqlbuild.compiler.planner.models import ChangeDetectionResult, SchemaFinding from sqlbuild.compiler.planner.types import ( BackfillAction, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py index 6ee1d5be87..24d986187f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.planner._helpers.output.strategy import resolve_schema_actions +from sqlbuild._native import resolve_schema_actions from sqlbuild.compiler.planner.models import SchemaAction, SchemaFinding from sqlbuild.compiler.planner.types import ( OnSchemaChange, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py index 24d7d390b6..34a4154bea 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.planner._helpers.output.strategy import build_model_warnings +from sqlbuild._native import build_model_warnings from sqlbuild.compiler.planner.models import ( ChangeDetectionResult, PlanWarning, From fca1633cdd21904b89df180def40070b4b97a224 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:28:47 +0100 Subject: [PATCH 32/40] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 9162367a28..104ae00f4d 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 53504 +remaining_lines = 52384 [[stays]] path = "executor" From 28377a82ea842dbca05362878a5acd5cf098bad6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:09:13 +0100 Subject: [PATCH 33/40] refactor: move the compile helper wrappers into native bindings --- .../src/assembly/project/constants.rs | 13 ++ .../src/assembly/project/main/mod.rs | 1 + .../project/main/sql_analysis_opt_out_help.rs | 16 +++ .../src/assembly/project/tests.rs | 2 + .../project/tests/test_opt_out_help.rs | 34 +++++ .../src/assembly/project/tests/test_types.rs | 6 + .../main/map_expanded_offset.rs | 10 ++ .../src/compiled_project/main/mod.rs | 1 + .../src/bindings/_helpers/analysis/mod.rs | 1 + .../_helpers/analysis/schema_request.rs | 66 +++++++++ .../analysis_session/fixture_columns.rs | 88 ++++++++++++ .../bindings/_helpers/analysis_session/mod.rs | 1 + .../_helpers/analysis_session/registration.rs | 4 +- .../analysis_session/stage_functions.rs | 2 +- .../compile_attachment/model_inputs.rs | 10 +- .../compile_driver/assembly_analysis.rs | 9 +- .../_helpers/compile_driver/assembly_facts.rs | 134 ++++++++++++++++++ .../compile_driver/assembly_project.rs | 59 ++------ .../compile_driver/assembly_sql_tests.rs | 117 +++++++++++++++ .../bindings/_helpers/compile_driver/mod.rs | 2 + .../compiled_project/expanded_offsets.rs | 44 ++++++ .../bindings/_helpers/compiled_project/mod.rs | 1 + .../_helpers/compiled_project/registration.rs | 1 + .../project_assembly/syntax_validation.rs | 19 +-- .../_helpers/scopes/relationship_names.rs | 55 ++++++- .../_helpers/semantic_checks/inputs.rs | 63 ++++---- .../_helpers/semantic_checks/resource_sql.rs | 2 +- .../semantic_shapes/source_bindings.rs | 56 +++----- fensu.toml | 5 + src/sqlbuild/_native.pyi | 13 ++ .../_helpers/analysis/binding_requests.py | 33 ----- .../_helpers/analysis/fixture_columns.py | 51 ------- .../_helpers/analysis/reference_names.py | 19 --- .../compile/_helpers/analysis/validation.py | 38 ----- .../compile/_helpers/diagnostics/recovery.py | 29 ---- .../_helpers/diagnostics/resource_sql.py | 23 --- .../_helpers/native_stages/assembly.py | 99 ------------- .../_helpers/native_stages/sql_tests.py | 58 -------- .../compiler/compile/_helpers/render/spans.py | 34 ----- .../compile/_helpers/sql_tests/core.py | 26 ---- .../compile/classes/sql_test_cte_extractor.py | 6 +- src/sqlbuild/compiler/compile/constants.py | 1 - .../compiler/compile/main/__init__.py | 1 - .../compile/main/_infer_fixture_columns.py | 20 --- .../compile/main/map_expanded_offset.py | 14 -- .../classes/fixture_column_inferences.py | 2 +- .../cli/commands/main/compile/helpers.py | 13 +- .../compile/test_native_analysis_seams.py | 2 +- .../compiler/analysis_session/helpers.py | 30 ++-- .../test_native_query_columns.py | 7 +- .../src/sqlbuild/compiler/pipeline/helpers.py | 14 +- .../pipeline/test_native_binding_catalog.py | 4 +- .../compiler/semantic_checks/helpers.py | 20 ++- .../test_native_semantic_checks.py | 2 +- .../compiler/sql_test_glue/helpers.py | 8 +- .../main/test_attempt_import_footprint.py | 4 +- .../compile/_helpers/test_expansion_spans.py | 5 +- .../compiler/compile/_helpers/test_spans.py | 9 +- .../_helpers/test_sql_analysis_columns.py | 7 +- .../compiler/planner/classes/helpers.py | 2 +- .../classes/test_fixture_column_inferences.py | 2 +- .../setting_help/main/test_setting_help.py | 7 +- .../src/sqlbuild/lint/test_expanded_lint.py | 3 +- 63 files changed, 760 insertions(+), 668 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/assembly/project/main/sql_analysis_opt_out_help.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/tests/test_opt_out_help.rs create mode 100644 crates/sqlbuild-analysis/src/compiled_project/main/map_expanded_offset.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/expanded_offsets.rs delete mode 100644 src/sqlbuild/compiler/compile/_helpers/analysis/binding_requests.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/analysis/reference_names.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/analysis/validation.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/render/spans.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py delete mode 100644 src/sqlbuild/compiler/compile/main/__init__.py delete mode 100644 src/sqlbuild/compiler/compile/main/_infer_fixture_columns.py delete mode 100644 src/sqlbuild/compiler/compile/main/map_expanded_offset.py diff --git a/crates/sqlbuild-analysis/src/assembly/project/constants.rs b/crates/sqlbuild-analysis/src/assembly/project/constants.rs index cb4c75ce95..91b7a61a5e 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/constants.rs @@ -47,3 +47,16 @@ pub(crate) const AUDIT_TARGET_KIND_ENUM: &str = "AttachedAuditTargetKind"; pub(crate) const ENVIRONMENT_READ_PREFIX: &str = "ENV:"; /// Characters that end an `ENV:` name besides Python whitespace. pub(crate) const ENVIRONMENT_NAME_TERMINATORS: [char; 6] = ['(', ')', ',', '\'', '"', '}']; +/// The MODEL header opt-out help of SQL the parser rejects. +pub(crate) const MODEL_OPT_OUT_HELP: &str = "if this SQL is valid for your warehouse, skip SQL \ + analysis for this model, add this to the MODEL header:\n MODEL (\n \ + sql_analysis false,\n ...\n );\n and please report the SQL \ + so the parser can support it"; +/// The report-only help of rejected SQL that no model header owns. +pub(crate) const REPORT_ONLY_HELP: &str = + "if this SQL is valid for your warehouse, please report it so the parser can support it"; +/// The project-wide and single-run opt-out helps, each after its own help label. +pub(crate) const PROJECT_AND_RUN_OPT_OUT_HELP: &str = "\n = help: SQL analysis is on for this \ + project; to turn it off for every model, set this in sqlbuild_project.toml:\n \ + [settings]\n sql_analysis = false\n = help: to skip SQL analysis for one run, \ + use `--no-sql-analysis`"; diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs index 9ac75e428e..9deee5d6ea 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs @@ -3,4 +3,5 @@ pub mod check_sql_syntax; pub mod configured_namespace; pub mod environment_template_names; pub mod preserved_logical_namespace; +pub mod sql_analysis_opt_out_help; pub mod sql_syntax_error; diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/sql_analysis_opt_out_help.rs b/crates/sqlbuild-analysis/src/assembly/project/main/sql_analysis_opt_out_help.rs new file mode 100644 index 0000000000..eb807a897b --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/main/sql_analysis_opt_out_help.rs @@ -0,0 +1,16 @@ +//! Help for SQL the parser rejects: the exact header, project and single-run opt-outs. + +use crate::assembly::project::constants::{ + MODEL_OPT_OUT_HELP, PROJECT_AND_RUN_OPT_OUT_HELP, REPORT_ONLY_HELP, +}; + +/// The opt-out help; only SQL a model owns offers the MODEL header entry. +#[must_use] +pub fn sql_analysis_opt_out_help(model_owned: bool) -> String { + let first: &str = if model_owned { + MODEL_OPT_OUT_HELP + } else { + REPORT_ONLY_HELP + }; + format!("{first}{PROJECT_AND_RUN_OPT_OUT_HELP}") +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests.rs b/crates/sqlbuild-analysis/src/assembly/project/tests.rs index 3ac1cfe81c..73b4356612 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests.rs @@ -4,6 +4,8 @@ mod configured_namespace; mod environment_names; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_opt_out_help.rs"] +mod opt_out_help; #[path = "tests/test_project_assembly.rs"] mod project_assembly; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_opt_out_help.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_opt_out_help.rs new file mode 100644 index 0000000000..7c3d4884bd --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_opt_out_help.rs @@ -0,0 +1,34 @@ +use crate::assembly::project::main::sql_analysis_opt_out_help::sql_analysis_opt_out_help; +use crate::assembly::project::tests::test_types::OptOutHelpTestCase; + +#[test] +fn given_ownership_when_rendering_opt_out_help_then_matches_python_help() { + let test_cases = [ + OptOutHelpTestCase { + description: "model-owned SQL offers the MODEL header entry", + model_owned: true, + expected_help: "if this SQL is valid for your warehouse, skip SQL analysis for this \ + model, add this to the MODEL header:\n MODEL (\n \ + sql_analysis false,\n ...\n );\n and please \ + report the SQL so the parser can support it\n = help: SQL analysis is on for \ + this project; to turn it off for every model, set this in \ + sqlbuild_project.toml:\n [settings]\n sql_analysis = \ + false\n = help: to skip SQL analysis for one run, use `--no-sql-analysis`", + }, + OptOutHelpTestCase { + description: "other SQL only asks for a report", + model_owned: false, + expected_help: "if this SQL is valid for your warehouse, please report it so the \ + parser can support it\n = help: SQL analysis is on for this project; to turn \ + it off for every model, set this in sqlbuild_project.toml:\n \ + [settings]\n sql_analysis = false\n = help: to skip SQL analysis \ + for one run, use `--no-sql-analysis`", + }, + ]; + + for test_case in test_cases { + let help: String = sql_analysis_opt_out_help(test_case.model_owned); + + assert_eq!(help, test_case.expected_help, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs index 5d6acb6b71..5be1373d69 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs @@ -92,3 +92,9 @@ pub(crate) struct ConfiguredNamespaceTestCase { pub(crate) value: &'static str, pub(crate) expected_namespace: Option<&'static str>, } + +pub(crate) struct OptOutHelpTestCase { + pub(crate) description: &'static str, + pub(crate) model_owned: bool, + pub(crate) expected_help: &'static str, +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/map_expanded_offset.rs b/crates/sqlbuild-analysis/src/compiled_project/main/map_expanded_offset.rs new file mode 100644 index 0000000000..dcfda6c06d --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/map_expanded_offset.rs @@ -0,0 +1,10 @@ +//! Resolve an offset in expanded SQL back onto the authored body. + +use crate::compiled_project::_helpers::locating::map_through_passes; +use crate::compiled_project::models::SplicedSpan; + +/// The offset in the first pass's source, and whether an expansion generated it. +#[must_use] +pub fn map_expanded_offset(offset: usize, passes: &[Vec]) -> (usize, bool) { + map_through_passes(offset, passes) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/mod.rs b/crates/sqlbuild-analysis/src/compiled_project/main/mod.rs index c9a5f9143b..f3b2e21870 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/main/mod.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/main/mod.rs @@ -5,6 +5,7 @@ pub mod declaration_error_message; pub mod enum_member_literal; pub mod header_metadata_keys; pub mod located_scan_error; +pub mod map_expanded_offset; pub mod merge_call_site_references; pub mod model_facts; pub mod opt_out_position; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs index b1cfc308a8..1e4d77872b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs @@ -10,4 +10,5 @@ pub(crate) mod queries; pub(crate) mod reference_facts; pub(crate) mod relation_names; mod required_columns; +pub(crate) mod schema_request; pub(crate) mod sql_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs new file mode 100644 index 0000000000..4d1ddd93fe --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs @@ -0,0 +1,66 @@ +//! Build the stable native schema-validation request of one model query. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_analysis::semantic_validation::main::normalize::normalize_analysis_sql; +use sqlbuild_analysis::semantic_validation::models::NormalizationInput; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::types::CompilerDetach; + +const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; +const GENERIC_DIALECT: &str = "generic"; + +/// The query, relation schemas and binding context of one schema-validation request. +pub(crate) struct SchemaRequestParts<'py> { + pub(crate) query_sql: String, + pub(crate) placeholders: Bound<'py, PyAny>, + pub(crate) dialect: Bound<'py, PyAny>, + pub(crate) schema: Bound<'py, PyAny>, + pub(crate) known_functions: Bound<'py, PyAny>, + pub(crate) known_types: Bound<'py, PyAny>, + pub(crate) catalog: Bound<'py, PyAny>, + pub(crate) quoted_identifiers_ignore_case: bool, +} + +/// A `SqlSchemaValidationRequest` over the normalized analysis SQL of `parts.query_sql`. +pub(crate) fn schema_validation_request<'py>( + py: Python<'py>, + parts: SchemaRequestParts<'py>, +) -> PyResult> { + let dialect: String = if parts.dialect.is_truthy()? { + parts.dialect.extract()? + } else { + GENERIC_DIALECT.to_owned() + }; + let placeholders: HashMap = if parts.placeholders.is_truthy()? { + parts.placeholders.extract()? + } else { + HashMap::new() + }; + let input: NormalizationInput = NormalizationInput { + sql: parts.query_sql, + dialect, + stubs: HashMap::new(), + placeholders, + }; + let sql: String = py + .compiler_detach(|| normalize_analysis_sql(input)) + .map_err(value_error)?; + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("sql", sql)?; + arguments.set_item("dialect", parts.dialect)?; + arguments.set_item("schema", parts.schema)?; + arguments.set_item("known_functions", parts.known_functions)?; + arguments.set_item("known_types", parts.known_types)?; + arguments.set_item( + "quoted_identifiers_ignore_case", + parts.quoted_identifiers_ignore_case, + )?; + arguments.set_item("catalog", parts.catalog)?; + py.import(SQL_ANALYSIS_MODELS_MODULE)? + .getattr("SqlSchemaValidationRequest")? + .call((), Some(&arguments)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs new file mode 100644 index 0000000000..28f000ad13 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs @@ -0,0 +1,88 @@ +//! Fixture-column inference: columns plus explicitly untyped NULL projections. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::analysis_session::main::projection_facts::projection_facts; +use sqlbuild_analysis::assembly::analysis_session::models::ProjectionFacts; +use sqlbuild_core::panics::main::native_failure::native_failure; +use sqlbuild_core::text::main::python_casefold::python_casefold; + +use crate::bindings::_helpers::analysis_session::query_columns::ColumnsRow; +use crate::bindings::_helpers::analysis_session::stage_functions::query_column_rows; +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::types::CompilerDetach; + +const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const PROJECTION_FACTS_CONTEXT: &str = "native fixture projection facts"; +const LEGACY_MODE: &str = "legacy"; + +/// `infer_fixture_column_facts(*, query_sql, inference_profile)`: a `FixtureColumnInference`. +#[pyfunction] +#[pyo3(signature = (**request))] +fn infer_fixture_column_facts<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("infer_fixture_column_facts", request)?; + let py: Python<'py> = request.py(); + let query_sql: String = request.get("query_sql")?.extract()?; + let profile: Bound<'py, PyAny> = request.get("inference_profile")?; + let query: Bound<'py, PyDict> = PyDict::new(py); + query.set_item("sql", &query_sql)?; + query.set_item("mode", LEGACY_MODE)?; + query.set_item("recover_cte_facts", false)?; + let query: Bound<'py, PyAny> = py + .import(ANALYSIS_MODELS_MODULE)? + .getattr("NativeColumnQuery")? + .call((), Some(&query))?; + let empty: Bound<'py, PyAny> = PyDict::new(py).into_any(); + let rows: Vec = query_column_rows( + PyTuple::new(py, [query])?.as_any(), + &profile, + (&empty, &empty), + )?; + let Some((true, Some(columns), false)) = rows.into_iter().next() else { + return Ok(py.None().into_bound(py)); + }; + let dialect: Option = profile.getattr("sql_analysis_dialect")?.extract()?; + let facts: ProjectionFacts = py + .compiler_detach(|| projection_facts(&query_sql, dialect.as_deref())) + .map_err(|reason| compiler_error(native_failure(PROJECTION_FACTS_CONTEXT, &reason)))?; + let simple_types: HashMap = facts.types.into_iter().collect(); + let column_class: Bound<'py, PyAny> = py + .import(COMPILE_MODELS_MODULE)? + .getattr("InferredColumn")?; + let nullability_class: Bound<'py, PyAny> = py + .import(LINEAGE_TYPES_MODULE)? + .getattr("InferredNullability")?; + let mut built: Vec> = Vec::with_capacity(columns.len()); + for (name, data_type, nullability) in columns { + let data_type: Option = data_type + .filter(|data_type| !data_type.is_empty()) + .or_else(|| simple_types.get(&python_casefold(&name)).cloned()); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("name", name)?; + arguments.set_item("type", data_type)?; + arguments.set_item("nullability", nullability_class.call1((nullability,))?)?; + built.push(column_class.call((), Some(&arguments))?); + } + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("columns", PyTuple::new(py, built)?)?; + arguments.set_item( + "null_literal_names", + PyFrozenSet::new(py, facts.null_literal_names)?, + )?; + arguments.set_item("quoted_names", PyFrozenSet::new(py, facts.quoted_names)?)?; + py.import(COMPILE_MODELS_MODULE)? + .getattr("FixtureColumnInference")? + .call((), Some(&arguments)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(infer_fixture_column_facts, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs index d212ed073a..baca587c27 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod fixture_columns; pub(crate) mod model_analyses; pub(crate) mod nullability_rules; pub(crate) mod oracle; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs index b0a64e9832..04b7a39550 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs @@ -3,11 +3,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::analysis_session::{ - model_analyses, oracle, outcome_objects, query_columns, session, stage_functions, + fixture_columns, model_analyses, oracle, outcome_objects, query_columns, session, + stage_functions, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { session::register(module)?; + fixture_columns::register(module)?; model_analyses::register(module)?; outcome_objects::register(module)?; query_columns::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs index 0414970f55..7843499028 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs @@ -243,7 +243,7 @@ fn or_empty<'py>( } } -fn query_column_rows( +pub(crate) fn query_column_rows( queries: &Bound<'_, PyAny>, profile: &Bound<'_, PyAny>, (types, nullability): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index dd1c8999c2..24d16e9b97 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -3,6 +3,7 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple}; +use sqlbuild_analysis::assembly::project::main::sql_analysis_opt_out_help::sql_analysis_opt_out_help; use crate::bindings::_helpers::attachments::cursor_intrinsics::reserved_cursor_markers; use crate::bindings::_helpers::compile_attachment::model_config::native_model_config_session; @@ -21,7 +22,6 @@ const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; -const VALIDATION_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.validation"; const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; const PROJECT_VARS_MODULE: &str = "sqlbuild._native"; @@ -42,11 +42,9 @@ fn dict<'py>( Ok(built) } -/// `sql_analysis_opt_out_help(model_owned=...)`. -fn opt_out_help<'py>(py: Python<'py>, model_owned: bool) -> PyResult> { - let keywords: Bound<'py, PyDict> = PyDict::new(py); - keywords.set_item("model_owned", model_owned)?; - imported(py, VALIDATION_MODULE, "sql_analysis_opt_out_help")?.call((), Some(&keywords)) +/// The parser opt-out help as a Python string. +fn opt_out_help(py: Python<'_>, model_owned: bool) -> PyResult> { + Ok(PyString::new(py, &sql_analysis_opt_out_help(model_owned)).into_any()) } /// The host the loop calls back into: `constant_text(*, declaration, file_path)`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index d8d224d721..d88b9f44d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -13,8 +13,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; -pub(crate) const NATIVE_STAGES_MODULE: &str = - "sqlbuild.compiler.compile._helpers.native_stages.assembly"; const ANALYSIS_CACHE_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.cache"; pub(crate) const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._record"; @@ -274,7 +272,7 @@ fn record_expression_shapes<'py>( } } let shapes: Vec> = call_with( - &imported(py, NATIVE_STAGES_MODULE, "expression_source_shapes")?, + &imported(py, NATIVE_MODULE, "infer_native_expression_source_shapes")?, &[ ("expressions", PyTuple::new(py, expressions)?.into_any()), ("profile", context.profile.clone()), @@ -386,7 +384,10 @@ fn model_analyses<'py>( ), ], )?; - imported(py, NATIVE_STAGES_MODULE, "analyze_model_sql")?.call1((analysis_request,)) + call_with( + &imported(py, NATIVE_MODULE, "analyze_native_model_sql")?, + &[("request", analysis_request)], + ) }) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs new file mode 100644 index 0000000000..a50bdb50a0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs @@ -0,0 +1,134 @@ +//! Every model's native facts and the project's resource facts, from the native stages. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::assembly_analysis::AnalysisContext; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const PROJECT_ASSEMBLY_MODELS_MODULE: &str = "sqlbuild.compiler.project_assembly.models"; + +/// `NativeProjectFacts` of `inputs`: resources, syntax validity, deps and standalone pivot proofs. +pub(crate) fn native_project_facts<'py>( + inputs: &Bound<'py, PyAny>, + context: &AnalysisContext<'py>, + names: &Bound<'py, PyAny>, + session: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let model_inputs: Bound<'py, PyAny> = inputs.getattr("model_inputs")?; + let dialect: Bound<'py, PyAny> = context.profile.getattr("sql_analysis_dialect")?; + let analysed: bool = !context.analyses.is_none(); + let succeeded: Bound<'py, PyAny> = if analysed { + context.analyses.getattr("succeeded")? + } else { + PyFrozenSet::empty(py)?.into_any() + }; + let positions: Bound<'py, PyAny> = if analysed { + context.analyses.getattr("positions")? + } else { + PyDict::new(py).into_any() + }; + let syntax_checks: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "model_syntax_checks")?, + &[ + ("model_inputs", model_inputs.clone()), + ("analysis_model_names", names.clone()), + ("analysis_succeeded", succeeded), + ], + )?; + let resources: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "assemble_native_project_resources")?, + &[ + ("inputs", inputs.clone()), + ("dialect", dialect.clone()), + ("syntax_checks", syntax_checks), + ], + )?; + let proofs: Bound<'py, PyDict> = pivot_proofs( + &call_with( + &imported(py, NATIVE_MODULE, "standalone_pivot_models")?, + &[ + ("model_inputs", model_inputs.clone()), + ("analysed", positions), + ], + )? + .cast_into::()?, + context, + session, + )?; + let model_facts: Bound<'py, PyAny> = + imported(py, PROJECT_ASSEMBLY_MODELS_MODULE, "NativeModelFacts")?; + let deps: Bound<'py, PyAny> = resources.getattr("model_deps")?; + let syntax_valid: Bound<'py, PyAny> = resources.getattr("model_syntax_valid")?; + let mut models: Vec> = Vec::new(); + for index in 0..model_inputs.len()? { + let proof: Bound<'py, PyAny> = match proofs.get_item(index)? { + Some(proof) => proof, + None => py.None().into_bound(py), + }; + models.push(call_with( + &model_facts, + &[ + ("deps", deps.get_item(index)?), + ("dynamic_contract", proof), + ("syntax_valid", syntax_valid.get_item(index)?), + ], + )?); + } + call_with( + &imported(py, PROJECT_ASSEMBLY_MODELS_MODULE, "NativeProjectFacts")?, + &[ + ("models", PyTuple::new(py, models)?.into_any()), + ("resources", resources), + ], + ) +} + +/// Each standalone pivot model's dynamic contract proof, by model position. +fn pivot_proofs<'py>( + pivots: &Bound<'py, PyDict>, + context: &AnalysisContext<'py>, + session: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = pivots.py(); + let tables: Bound<'py, PyAny> = call_with( + &imported(py, ANALYSIS_MODELS_MODULE, "NativePivotTables")?, + &[ + ("dialect", context.profile.getattr("sql_analysis_dialect")?), + ("column_types_by_table", context.column_types.clone()), + ( + "authoritative_column_types_by_table", + context.complete_schemas.clone(), + ), + ("column_nullability_by_table", context.nullability.clone()), + ( + "dynamic_families_by_table", + context.dynamic_families.clone(), + ), + ], + )?; + let proven: Vec> = call_with( + &imported(py, NATIVE_MODULE, "prove_native_dynamic_contracts")?, + &[ + ("session", session.clone()), + ("tables", tables), + ("models", PyTuple::new(py, pivots.values())?.into_any()), + ], + )? + .try_iter()? + .collect::>()?; + if proven.len() != pivots.len() { + return Err(PyValueError::new_err( + "native pivot proofs do not match the standalone pivot models", + )); + } + let proofs: Bound<'py, PyDict> = PyDict::new(py); + for (index, proof) in pivots.keys().iter().zip(proven) { + proofs.set_item(index, proof)?; + } + Ok(proofs) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index 16a0ce90e3..28e30a54b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -1,20 +1,20 @@ //! Compiled project assembly: the project's models, resources, checks and semantic diagnostics. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTuple}; use sqlbuild_analysis::assembly::project::main::configured_namespace::configured_namespace; use crate::bindings::_helpers::compile_driver::assembly::AssemblyRequest; use crate::bindings::_helpers::compile_driver::assembly_analysis::{ - AnalysisContext, NATIVE_STAGES_MODULE, analysis_context, and_not, -}; -use crate::bindings::_helpers::compile_driver::assembly_checks::{ - compiled_sql_scenarios, test_model_query_overrides, + AnalysisContext, analysis_context, and_not, }; +use crate::bindings::_helpers::compile_driver::assembly_checks::compiled_sql_scenarios; +use crate::bindings::_helpers::compile_driver::assembly_facts::native_project_facts; use crate::bindings::_helpers::compile_driver::assembly_models::{ binding_rows, compiled_models, project_binding_diagnostics, }; +use crate::bindings::_helpers::compile_driver::assembly_sql_tests::assembled_sql_tests; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::model_config::template_data::early_model_templates; use crate::bindings::_helpers::scopes::scope_errors::report_scope_index_errors; @@ -24,8 +24,6 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; pub(crate) const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTER_TYPES_MODULE: &str = "sqlbuild.adapter.contract.types"; const SCENARIO_CONFIG_MODULE: &str = "sqlbuild._native"; -const SQL_TESTS_MODULE: &str = "sqlbuild.compiler.compile._helpers.native_stages.sql_tests"; -const RECOVERY_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.recovery"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const SOURCE_SLOT: &str = "source"; const SEED_SLOT: &str = "seed"; @@ -82,26 +80,7 @@ pub(crate) fn assembled_project<'py>( } else { context.analyses.getattr("session")? }; - let native: Bound<'py, PyAny> = call_with( - &imported(py, NATIVE_STAGES_MODULE, "native_project_facts")?, - &[ - ("inputs", inputs.clone()), - ("dialect", context.profile.getattr("sql_analysis_dialect")?), - ("analysis_model_names", names.clone()), - ("analyses", context.analyses.clone()), - ("session", native_session.clone()), - ("column_types_by_table", context.column_types.clone()), - ( - "authoritative_column_types_by_table", - context.complete_schemas.clone(), - ), - ("column_nullability_by_table", context.nullability.clone()), - ( - "dynamic_families_by_table", - context.dynamic_families.clone(), - ), - ], - )?; + let native: Bound<'py, PyAny> = native_project_facts(inputs, &context, names, &native_session)?; let resources: Bound<'py, PyAny> = native.getattr("resources")?; let scope_index: Bound<'py, PyAny> = scope_index_with_compile_usages(inputs)?; report_scope_index_errors(&scope_index)?; @@ -117,7 +96,7 @@ pub(crate) fn assembled_project<'py>( let project: Bound<'py, PyAny> = project_view(inputs, &fields, &models)?; let binding_results: Bound<'py, PyDict> = binding_rows(&context.analyses)?; call_with( - &imported(py, RECOVERY_MODULE, "complete_semantic_diagnostics")?, + &imported(py, NATIVE_MODULE, "complete_semantic_project")?, &[ ("project", project), ("profile", context.profile.clone()), @@ -128,7 +107,7 @@ pub(crate) fn assembled_project<'py>( &inputs.getattr("no_sql_validation")?, )?, ), - ("native_session", native_session), + ("session", native_session), ("binding_results", binding_results.into_any()), ], ) @@ -270,27 +249,7 @@ fn project_fields<'py>( ] { fields.set_item(name, compiled_resources(inputs, resources, kind)?)?; } - let inputs_owned: pyo3::Py = inputs.clone().unbind(); - let overrides: Bound<'py, PyCFunction> = PyCFunction::new_closure( - py, - None, - None, - move |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| { - let py: Python<'_> = arguments.py(); - test_model_query_overrides(&arguments.get_item(0)?, inputs_owned.bind(py)) - .map(|overrides| overrides.into_any().unbind()) - }, - )?; - fields.set_item( - "sql_tests", - call_with( - &imported(py, SQL_TESTS_MODULE, "assemble_sql_tests")?, - &[ - ("inputs", inputs.clone()), - ("macro_mock_queries", overrides.into_any()), - ], - )?, - )?; + fields.set_item("sql_tests", assembled_sql_tests(inputs)?)?; fields.set_item("sql_scenarios", compiled_sql_scenarios(inputs)?)?; for name in ["loader_functions", "hook_functions"] { fields.set_item(name, discovered.getattr(name)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs new file mode 100644 index 0000000000..40601cab98 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs @@ -0,0 +1,117 @@ +//! Every SQL test assembled natively; macro-mocked tests' model queries run the user's macros. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyList, PyTuple}; + +use crate::bindings::_helpers::boundary::panics::NativeCompilerError; +use crate::bindings::_helpers::compile_driver::assembly_checks::test_model_query_overrides; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const DECIMAL_MODULE: &str = "decimal"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const SQL_TEST_INPUT_FAILURE: &str = "input"; +const SQL_TEST_DECIMAL_OVERFLOW: &str = "decimal_overflow"; + +/// Every compiled SQL test, raising each assembly failure in the tests' order. +pub(crate) fn assembled_sql_tests<'py>( + inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let assemblies: Vec> = call_with( + &imported(py, NATIVE_MODULE, "assemble_native_sql_tests")?, + &[("inputs", inputs.clone())], + )? + .try_iter()? + .collect::>()?; + let test_inputs: Vec> = inputs + .getattr("test_inputs")? + .try_iter()? + .collect::>()?; + if assemblies.len() != test_inputs.len() { + return Err(NativeCompilerError::new_err( + "native SQL test assembly returned a different number of tests", + )); + } + let report: Bound<'py, PyAny> = imported(py, COLLECTOR_MODULE, "report_compile_diagnostic")?; + let mocked_input: Bound<'py, PyAny> = + imported(py, COMPILE_MODELS_MODULE, "CompileModelSqlTestInputPayload")?; + let mut tests: Vec> = Vec::with_capacity(assemblies.len()); + for (test_input, assembly) in test_inputs.iter().zip(assemblies) { + let failure: Bound<'py, PyAny> = assembly.getattr("failure")?; + let kind: Option = if failure.is_none() { + None + } else { + Some(failure.getattr("kind")?.extract()?) + }; + let mut test: Bound<'py, PyAny> = assembly.getattr("test")?; + if test.is_none() { + return Err(assembly_error(py, &failure, kind.as_deref())?); + } + for item in assembly.getattr("diagnostics")?.try_iter()? { + let (key, diagnostic): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + call_with(&report, &[("key", key), ("diagnostic", diagnostic)])?; + } + let payload: Bound<'py, PyAny> = test_input.getattr("payload")?; + if payload.is_instance(&mocked_input)? && payload.getattr("macro_mocks")?.is_truthy()? { + test = with_model_queries( + &test, + test_model_query_overrides(test_input, inputs)?.as_any(), + )?; + } + if kind.as_deref() == Some(SQL_TEST_DECIMAL_OVERFLOW) { + let overflow: Bound<'py, PyAny> = imported(py, DECIMAL_MODULE, "Overflow")?; + let raised: Bound<'py, PyAny> = overflow.call1((PyList::new(py, [&overflow])?,))?; + return Err(PyErr::from_value(raised)); + } + tests.push(test); + } + PyTuple::new(py, tests) +} + +/// The failure of a test that did not assemble: input errors as `CompileInputError`. +fn assembly_error( + py: Python<'_>, + failure: &Bound<'_, PyAny>, + kind: Option<&str>, +) -> PyResult { + let message: String = if failure.is_none() { + String::new() + } else { + failure.getattr("message")?.extract()? + }; + if kind == Some(SQL_TEST_INPUT_FAILURE) { + let error: Bound<'_, PyAny> = + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?.call1((message,))?; + return Ok(PyErr::from_value(error)); + } + Ok(NativeCompilerError::new_err(message)) +} + +/// `test` with its model payload's query overrides set to `queries`. +fn with_model_queries<'py>( + test: &Bound<'py, PyAny>, + queries: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = test.py(); + let payload: Bound<'py, PyAny> = test.getattr("payload")?; + if !payload.is_instance(&imported( + py, + COMPILE_MODELS_MODULE, + "CompiledModelSqlTestPayload", + )?)? { + return Ok(test.clone()); + } + let replace: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?; + let payload: Bound<'py, PyAny> = replace.call( + (payload,), + Some(&keywords( + py, + &[("model_query_overrides", queries.clone())], + )?), + )?; + replace.call((test,), Some(&keywords(py, &[("payload", payload)])?)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index bbda2f8952..102f4a0e04 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -2,8 +2,10 @@ pub(crate) mod analysis; pub(crate) mod assembly; pub(crate) mod assembly_analysis; pub(crate) mod assembly_checks; +pub(crate) mod assembly_facts; pub(crate) mod assembly_models; pub(crate) mod assembly_project; +pub(crate) mod assembly_sql_tests; pub(crate) mod command; pub(crate) mod config_validation; pub(crate) mod deferral; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/expanded_offsets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/expanded_offsets.rs new file mode 100644 index 0000000000..b7a8abb148 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/expanded_offsets.rs @@ -0,0 +1,44 @@ +//! Map offsets in expanded SQL back onto the authored text that produced them. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::compiled_project::main::map_expanded_offset::map_expanded_offset as mapped; +use sqlbuild_analysis::compiled_project::models::SplicedSpan; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; + +/// `map_expanded_offset(*, offset, passes)`: a `MappedOffset` in the first pass's source. +#[pyfunction] +#[pyo3(signature = (*, offset, passes))] +fn map_expanded_offset<'py>( + py: Python<'py>, + offset: usize, + passes: &Bound<'py, PyAny>, +) -> PyResult> { + let mut spans: Vec> = Vec::new(); + for pass in passes.try_iter()? { + let mut pass_spans: Vec = Vec::new(); + for span in pass?.try_iter()? { + let span: Bound<'py, PyAny> = span?; + pass_spans.push(SplicedSpan { + source_start: span.getattr("source_start")?.extract()?, + source_end: span.getattr("source_end")?.extract()?, + output_start: span.getattr("output_start")?.extract()?, + output_end: span.getattr("output_end")?.extract()?, + }); + } + spans.push(pass_spans); + } + let (offset, generated) = mapped(offset, &spans); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("offset", offset)?; + arguments.set_item("generated", generated)?; + py.import(COMPILE_MODELS_MODULE)? + .getattr("MappedOffset")? + .call((), Some(&arguments)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(map_expanded_offset, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index 29c1c178b1..802f7dc128 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod audit_loop; pub(crate) mod audit_views; pub(crate) mod compiled_model_analysis; pub(crate) mod compiled_model_view; +pub(crate) mod expanded_offsets; pub(crate) mod function_loop; pub(crate) mod function_views; pub(crate) mod inputs_view; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs index 93b6bc7006..826d439918 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs @@ -4,5 +4,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::compiled_project::project::register(module)?; crate::bindings::_helpers::compiled_project::model_loop::register(module)?; crate::bindings::_helpers::compiled_project::attachment::register(module)?; + crate::bindings::_helpers::compiled_project::expanded_offsets::register(module)?; crate::bindings::_helpers::compiled_project::records::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs index 3e27210052..b6b7f7fe44 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs @@ -7,6 +7,7 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::project::main::sql_analysis_opt_out_help::sql_analysis_opt_out_help; use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; use crate::bindings::_helpers::project_assembly::resources::syntax_error_message; @@ -16,8 +17,6 @@ const SQL_HOOK_ENTRY: &str = "SqlHookEntry"; const PYTHON_HOOK_ENTRY: &str = "PythonHookEntry"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_INPUT_ERROR: &str = "CompileInputError"; -const VALIDATION_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.validation"; -const OPT_OUT_HELP: &str = "sql_analysis_opt_out_help"; const GENERIC_DIALECT: &str = "generic"; /// `(placeholder defaults, dialect)` a statement parses with. @@ -43,14 +42,8 @@ struct HookSite<'a, 'py> { /// `CompileInputError(message, help=sql_analysis_opt_out_help(model_owned=True))`. fn syntax_error(py: Python<'_>, message: String) -> PyErr { let raised = || -> PyResult { - let help_arguments: Bound<'_, PyDict> = PyDict::new(py); - help_arguments.set_item("model_owned", true)?; - let help: Bound<'_, PyAny> = py - .import(VALIDATION_MODULE)? - .getattr(OPT_OUT_HELP)? - .call((), Some(&help_arguments))?; let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("help", help)?; + keywords.set_item("help", sql_analysis_opt_out_help(true))?; let error: Bound<'_, PyAny> = py .import(COMPILE_ERRORS_MODULE)? .getattr(COMPILE_INPUT_ERROR)? @@ -248,7 +241,15 @@ pub(crate) fn hook_sql_statements(value: Bound<'_, PyAny>) -> PyResult String { + sql_analysis_opt_out_help(model_owned) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(opt_out_help, module)?)?; module.add_function(wrap_pyfunction!(validate_hook_sql_syntax, module)?)?; module.add_function(wrap_pyfunction!(validate_sql_syntax, module)?)?; module.add_function(wrap_pyfunction!(hook_sql_statements, module)?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs index 8345733988..6c12986892 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs @@ -1,6 +1,9 @@ //! Expected-model relationship names and top-level CTEs of SQL tests and scenarios, scanned natively. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_scopes::relationship_names::main::expected_model_names::expected_model_names; use sqlbuild_scopes::relationship_names::main::top_level_ctes::scan_top_level_ctes; @@ -13,6 +16,9 @@ use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use crate::bindings::types::CompilerDetach; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; + /// One scan's error, or its values. type Scanned = (Option, Vec); @@ -72,7 +78,54 @@ fn scope_test_ctes( .collect()) } +/// One SQL test body's raw top-level CTEs; a scan failure raises `CompileInputError`. +#[pyfunction] +#[pyo3(signature = (*, sql, file_label, syntax))] +fn extract_sql_test_ctes<'py>( + py: Python<'py>, + sql: &str, + file_label: &str, + syntax: &Bound<'py, PyAny>, +) -> PyResult> { + let syntax: LexicalSyntax = syntax + .getattr("native_mapping")? + .extract::()? + .into(); + let outcome: TopLevelCtes = py + .compiler_detach(|| { + Ok(scan_top_level_ctes( + sql, + file_label, + RelationshipSource::Test, + &syntax, + )) + }) + .map_err(compiler_error)?; + let ctes: Vec<(String, String)> = match outcome { + TopLevelCtes::Scanned(ctes) => ctes, + TopLevelCtes::Failed(message) => { + let error: Bound<'py, PyAny> = py + .import(COMPILE_ERRORS_MODULE)? + .getattr("CompileInputError")? + .call1((message,))?; + return Err(PyErr::from_value(error)); + } + }; + let cte_class: Bound<'py, PyAny> = py + .import(COMPILE_MODELS_MODULE)? + .getattr("CompileSqlTestCte")?; + let mut built: Vec> = Vec::with_capacity(ctes.len()); + for (name, body) in ctes { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("name", name)?; + arguments.set_item("sql_body", body)?; + built.push(cte_class.call((), Some(&arguments))?); + } + PyTuple::new(py, built) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(extract_sql_test_ctes, module)?)?; module.add_function(wrap_pyfunction!(scope_expected_model_names, module)?)?; module.add_function(wrap_pyfunction!(scope_test_ctes, module)?)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs index f2eddbcf9e..273c987b1a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs @@ -1,17 +1,18 @@ //! Compile-owned inputs of the semantic completion stage: shapes, requests and SQL test columns. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; use crate::bindings::_helpers::analysis::binding_catalog::ignores_quoted_case; +use crate::bindings::_helpers::analysis::schema_request::{ + SchemaRequestParts, schema_validation_request, +}; +use crate::bindings::_helpers::semantic_checks::resource_sql::resource_sql_diagnostics_of; use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; +use crate::bindings::models::ProjectCatalog; const NATIVE_MODULE: &str = "sqlbuild._native"; const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; -const BINDING_REQUESTS_MODULE: &str = - "sqlbuild.compiler.compile._helpers.analysis.binding_requests"; -const RESOURCE_SQL_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.resource_sql"; -const DATACLASSES_MODULE: &str = "dataclasses"; /// The column names a metadata config value lists: a name, or the names of a list or tuple. pub(crate) fn column_names(value: &Bound<'_, PyAny>) -> PyResult> { @@ -106,30 +107,20 @@ pub(crate) fn revalidation_request<'py>( }; schema.set_item(name, shape)?; } - let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("query_sql", model.getattr("query_sql")?)?; - arguments.set_item("placeholders", py.None())?; - arguments.set_item("dialect", &dialect)?; - arguments.set_item( - "known_functions", - PyTuple::new(py, function_names(&functions)?)?, - )?; - arguments.set_item( - "known_types", - PyTuple::new(py, declared_types(&functions, shapes.as_any())?)?, - )?; - arguments.set_item("binding_schema", schema)?; - let request: Bound<'py, PyAny> = py - .import(BINDING_REQUESTS_MODULE)? - .getattr("get_complete_schema_binding_request")? - .call((), Some(&arguments))?; let ignores: bool = ignores_quoted_case(&project.getattr("effective_connection")?, &dialect)?; - let replaced: Bound<'py, PyDict> = PyDict::new(py); - replaced.set_item("catalog", project.getattr("binding_catalog")?)?; - replaced.set_item("quoted_identifiers_ignore_case", ignores)?; - py.import(DATACLASSES_MODULE)? - .getattr("replace")? - .call((request,), Some(&replaced)) + schema_validation_request( + py, + SchemaRequestParts { + query_sql: model.getattr("query_sql")?.extract()?, + placeholders: py.None().into_bound(py), + known_functions: PyTuple::new(py, function_names(&functions)?)?.into_any(), + known_types: PyTuple::new(py, declared_types(&functions, shapes.as_any())?)?.into_any(), + dialect, + schema: schema.into_any(), + catalog: project.getattr("binding_catalog")?, + quoted_identifiers_ignore_case: ignores, + }, + ) } /// The compile-time SQL checks of audits, SQL tests and SQL hooks. @@ -138,12 +129,12 @@ pub(crate) fn resource_sql_diagnostics<'py>( shapes: &Bound<'py, PyAny>, profile: &Bound<'py, PyAny>, ) -> PyResult> { - let py: Python<'py> = project.py(); - let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("project", project)?; - arguments.set_item("shapes", shapes)?; - arguments.set_item("profile", profile)?; - py.import(RESOURCE_SQL_MODULE)? - .getattr("get_resource_sql_diagnostics")? - .call((), Some(&arguments)) + let native: Bound<'py, PyAny> = project.getattr("binding_catalog")?.getattr("native")?; + Ok(resource_sql_diagnostics_of( + native.extract::>()?, + project, + shapes.cast::()?, + profile, + )? + .into_any()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index 5fb50715ed..d781c15a56 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -36,7 +36,7 @@ struct DiagnosticClasses<'py> { /// Bind the project's audit, SQL test and SQL hook bodies on `catalog` against `shapes`. #[pyfunction] -fn resource_sql_diagnostics_of<'py>( +pub(crate) fn resource_sql_diagnostics_of<'py>( catalog: PyRef<'_, ProjectCatalog>, project: &Bound<'py, PyAny>, shapes: &Bound<'py, PyDict>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs index 75420b0b46..b705744b91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs @@ -9,6 +9,9 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::analysis::binding_catalog::{ self, CASE_SENSITIVE_BINDING_DIALECTS, NATIVE_DIALECT_ALIASES, schema_validations, }; +use crate::bindings::_helpers::analysis::schema_request::{ + SchemaRequestParts, schema_validation_request, +}; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; @@ -22,8 +25,6 @@ const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const BINDING_UNKNOWN_TABLE_INTERNAL_CODE: &str = "B000"; -const BINDING_REQUESTS_MODULE: &str = - "sqlbuild.compiler.compile._helpers.analysis.binding_requests"; const DATACLASSES_MODULE: &str = "dataclasses"; const GRAPHLIB_MODULE: &str = "graphlib"; const REF_KIND: &str = "ref"; @@ -266,41 +267,30 @@ fn validated_diagnostics<'py>( let known_functions: Object<'py> = PyTuple::new(py, function_names(&functions)?)?.into_any(); let known_types: Object<'py> = PyTuple::new(py, declared_types(&functions, shapes.as_any())?)?.into_any(); - let request_builder: Object<'py> = py - .import(BINDING_REQUESTS_MODULE)? - .getattr("get_complete_schema_binding_request")?; - let ignores: Object<'py> = ignores_quoted_case(project, dialect)?; + let ignores: bool = + binding_catalog::ignores_quoted_case(&project.getattr("effective_connection")?, dialect)?; let mut requests: Vec> = Vec::with_capacity(models.len()); for model in models { - let query_sql: Object<'py> = PyString::new( + let query_sql: String = analysis_cursor_sql( py, - &analysis_cursor_sql( - py, - &model.getattr("query_sql")?.extract::()?, - &model - .getattr("config")? - .getattr("values")? - .call_method1("get", ("cursor_type",))?, - )?, - ) - .into_any(); - let request: Object<'py> = call( - &request_builder, - &[ - ("known_functions", known_functions.clone()), - ("known_types", known_types.clone()), - ("query_sql", query_sql), - ("placeholders", placeholders(model)?), - ("dialect", dialect.clone()), - ("binding_schema", model_inputs(model, shapes)?.into_any()), - ], + &model.getattr("query_sql")?.extract::()?, + &model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("cursor_type",))?, )?; - requests.push(replaced( - &request, - &[ - ("catalog", catalog.clone()), - ("quoted_identifiers_ignore_case", ignores.clone()), - ], + requests.push(schema_validation_request( + py, + SchemaRequestParts { + query_sql, + placeholders: placeholders(model)?, + dialect: dialect.clone(), + schema: model_inputs(model, shapes)?.into_any(), + known_functions: known_functions.clone(), + known_types: known_types.clone(), + catalog: catalog.clone(), + quoted_identifiers_ignore_case: ignores, + }, )?); } let results: Object<'py> = diff --git a/fensu.toml b/fensu.toml index 005136f41a..7249edb97e 100644 --- a/fensu.toml +++ b/fensu.toml @@ -88,6 +88,11 @@ rule = "FFR309" path = "src/sqlbuild/compiler/fingerprints/__init__.py" reason = "Fingerprint hashing and storage run natively; the package keeps the stored-row models, column and node-type vocabulary adapters and the executor share." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/compile/__init__.py" +reason = "Compilation runs natively; the package keeps the compile models, views, constants, errors, the diagnostic collector and the user-macro and Python-resource glue the native compiler calls back into." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/refactoring/__init__.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 7560db38f3..78bb2a5ee0 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -94,6 +94,7 @@ from sqlbuild.compiler.compile.models import ( CompileSqlFunctionInput, CompileSqlReference, CompileSqlScenarioInput, + CompileSqlTestCte, CompileSqlTestInput, CursorInputRoles, DeclarationResolutionContext, @@ -101,12 +102,14 @@ from sqlbuild.compiler.compile.models import ( DeclarationScopeResolver, DynamicColumnContractProof, ExpansionSpan, + FixtureColumnInference, FunctionArgument, FunctionReturnColumn, InferredColumn, LoadedMacro, MacroContext, MacroExpansionResult, + MappedOffset, ModelInputBuildContext, NativeModelConfigInputs, NativeModelConfigSession, @@ -2823,6 +2826,9 @@ def parse_cli_arguments( argv: list[str], defaults: list[tuple[str, object]], *, use_color: bool, columns: int ) -> tuple[list[tuple[str, object]] | None, int | None]: ... def run_cli(argv: Sequence[str] | None = None, handlers: object | None = None) -> int: ... +def extract_sql_test_ctes( + *, sql: str, file_label: str, syntax: SqlLexicalSyntax +) -> tuple[CompileSqlTestCte, ...]: ... def scope_test_ctes( texts: list[tuple[str, str]], syntax: dict[str, object] ) -> list[tuple[str | None, list[tuple[str, str]]]]: ... @@ -2860,6 +2866,12 @@ def substitute_placeholder_defaults(sql: str, placeholders: list[tuple[str, str] def fixture_projection_facts( sql: str, dialect: str | None, / ) -> tuple[list[tuple[str, str]], list[str], list[str]]: ... +def map_expanded_offset( + *, offset: int, passes: Sequence[Sequence[ExpansionSpan]] +) -> MappedOffset: ... +def infer_fixture_column_facts( + *, query_sql: str, inference_profile: ExpressionInferenceProfile +) -> FixtureColumnInference | None: ... def find_macro_call_names(sql: str, /) -> tuple[str, ...]: ... def build_native_declaration_contexts( *, @@ -3282,6 +3294,7 @@ def serialize_hook_entries( def named_declaration_role_kind(*, relative_path: PurePath) -> DeclarationKind | None: ... def discover_model_description_inputs(*, project_dir: Path) -> DiscoveredProjectInputs: ... def microbatch_guidance() -> tuple[str, str]: ... +def sql_analysis_opt_out_help(*, model_owned: bool = True) -> str: ... def explicit_references_help(*, allowed: str) -> str: ... def run_format_command( *, diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/binding_requests.py b/src/sqlbuild/compiler/compile/_helpers/analysis/binding_requests.py deleted file mode 100644 index 35c2a57dbd..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/binding_requests.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Native schema-validation requests for model SQL.""" - -from __future__ import annotations - -from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql -from sqlbuild.compiler.sql_analysis.models import ( - SqlSchemaValidationRequest, -) - - -def get_complete_schema_binding_request( - *, - query_sql: str, - placeholders: dict[str, str] | None, - dialect: str | None, - binding_schema: dict[str, dict[str, str]], - known_functions: tuple[str, ...] = (), - known_types: tuple[str, ...] = (), - cleaned_sql: str | None = None, -) -> SqlSchemaValidationRequest: - """Build one stable native schema-validation request.""" - - if cleaned_sql is None: - cleaned_sql = normalize_analysis_sql( - sql=query_sql, dialect=dialect, placeholders=placeholders - ) - return SqlSchemaValidationRequest( - sql=cleaned_sql, - dialect=dialect, - schema=binding_schema, - known_functions=known_functions, - known_types=known_types, - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py b/src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py deleted file mode 100644 index 1b7373d28d..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Fixture-column inference implementation.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild._native import infer_native_query_columns -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns -from sqlbuild.compiler.compile.models import ( - FixtureColumnInference, - InferredColumn, -) - - -def infer_fixture_column_facts( - *, query_sql: str, inference_profile: ExpressionInferenceProfile -) -> FixtureColumnInference | None: - """Infer fixture columns and identify explicitly untyped NULL projections.""" - - analysis: NativeQueryColumns = infer_native_query_columns( - queries=(NativeColumnQuery(sql=query_sql, mode="legacy", recover_cte_facts=False),), - profile=inference_profile, - )[0] - if not analysis.succeeded or analysis.has_star or analysis.columns is None: - return None - simple_types, null_literal_names, quoted_names = _simple_projection_facts( - query_sql=query_sql, - dialect=inference_profile.sql_analysis_dialect, - ) - return FixtureColumnInference( - columns=tuple( - InferredColumn( - name=column.name, - type=column.type or simple_types.get(column.name.casefold()), - nullability=column.nullability, - ) - for column in analysis.columns - ), - null_literal_names=null_literal_names, - quoted_names=quoted_names, - ) - - -def _simple_projection_facts( - *, query_sql: str, dialect: str | None -) -> tuple[dict[str, str], frozenset[str], frozenset[str]]: - types: list[tuple[str, str]] - null_literal_names: list[str] - quoted_names: list[str] - types, null_literal_names, quoted_names = _native.fixture_projection_facts(query_sql, dialect) - return dict(types), frozenset(null_literal_names), frozenset(quoted_names) diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/reference_names.py b/src/sqlbuild/compiler/compile/_helpers/analysis/reference_names.py deleted file mode 100644 index 7f929085c3..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/reference_names.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Reference analysis names, lineage resource types, qualified-reference scans and placeholders.""" - -from __future__ import annotations - -import sqlbuild._native as _native - - -def table_function_analysis_name(function_name: str) -> str: - """Return the stable relation stub used to analyze a table-function call.""" - - return f"__sqlbuild_table_function_{function_name}" - - -def substitute_placeholder_defaults(*, query_sql: str, placeholders: dict[str, str]) -> str: - """Replace @@@name tokens with their default values for SQL analysis parsing.""" - - if not placeholders: - return query_sql - return _native.substitute_placeholder_defaults(query_sql, list(placeholders.items())) diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py b/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py deleted file mode 100644 index 6265087cef..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Help for SQL the parser rejects.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.constants import SQL_ANALYSIS_OPT_OUT_ENTRY -from sqlbuild.compiler.discovery.constants import ( - PROJECT_CONFIG_FILENAME, - SETTINGS_SECTION, - SQL_ANALYSIS_CONFIG_KEY, -) -from sqlbuild.errors.setting_help.main.join_helps import join_helps -from sqlbuild.errors.setting_help.main.model_header_help import model_header_help -from sqlbuild.errors.setting_help.main.setting_help import setting_help - - -def sql_analysis_opt_out_help(*, model_owned: bool = True) -> str: - """Help for SQL the parser rejects: the exact header, project and single-run opt-outs.""" - - report: str = ( - "if this SQL is valid for your warehouse, please report it so the parser can support it" - ) - return join_helps( - model_header_help( - purpose="if this SQL is valid for your warehouse, skip SQL analysis for this model", - entry=SQL_ANALYSIS_OPT_OUT_ENTRY, - follow_up="and please report the SQL so the parser can support it", - ) - if model_owned - else report, - setting_help( - purpose="SQL analysis is on for this project; to turn it off for every model", - file_name=PROJECT_CONFIG_FILENAME, - section=SETTINGS_SECTION, - key=SQL_ANALYSIS_CONFIG_KEY, - value=False, - ), - "to skip SQL analysis for one run, use `--no-sql-analysis`", - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py deleted file mode 100644 index 907ed49485..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Complete semantic diagnostics: type recovery, metadata checks, recovery and explanations.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild._native import complete_semantic_project -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.types import DiagnosticRow -from sqlbuild.compiler.compile.types import CompiledProjectView - - -def complete_semantic_diagnostics( - *, - project: CompiledProjectView, - profile: ExpressionInferenceProfile, - binding_results: dict[str, list[DiagnosticRow]], - resource_sql_analysis: bool = True, - native_session: Any | None = None, -) -> CompiledProjectView: - """Recover type facts before metadata checks, then explain retained root diagnostics.""" - - return complete_semantic_project( - project=project, - profile=profile, - binding_results=binding_results, - resource_sql_analysis=resource_sql_analysis, - session=native_session, - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py deleted file mode 100644 index c04e76824c..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Hand audit, SQL test and SQL hook SQL to the native resource SQL checks.""" - -from __future__ import annotations - -from typing import Any, cast - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import CompiledProjectView - - -def get_resource_sql_diagnostics( - *, - project: CompiledProjectView, - shapes: dict[str, dict[str, str]], - profile: ExpressionInferenceProfile, -) -> tuple[CompilerDiagnostic, ...]: - """Bind audits, SQL tests and SQL hooks against the same closed shapes as models.""" - - return _native.resource_sql_diagnostics_of( - cast(Any, project.binding_catalog).native, project, shapes, profile - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py deleted file mode 100644 index 73283c8cb8..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Hand compiled-project assembly and model analysis to their native stages when enabled.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild._native import ( - analyze_native_model_sql, - assemble_native_project_resources, - infer_native_expression_source_shapes, - model_syntax_checks, - prove_native_dynamic_contracts, - standalone_pivot_models, -) -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.models import ( - NativeModelAnalyses, - NativeModelAnalysisRequest, - NativePivotTables, -) -from sqlbuild.compiler.analysis_session.types import ProofRow -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.project_assembly.models import ( - NativeModelFacts, - NativeProjectFacts, - NativeProjectResources, -) -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -def native_project_facts( - *, - inputs: CompileInputsView, - dialect: str | None, - analysis_model_names: frozenset[str] | None, - analyses: NativeModelAnalyses | None, - session: Any | None, - column_types_by_table: dict[str, dict[str, str]], - authoritative_column_types_by_table: dict[str, dict[str, str]], - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], -) -> NativeProjectFacts: - """Return the native project facts.""" - - resources: NativeProjectResources = assemble_native_project_resources( - inputs=inputs, - dialect=dialect, - syntax_checks=model_syntax_checks( - model_inputs=inputs.model_inputs, - analysis_model_names=analysis_model_names, - analysis_succeeded=frozenset() if analyses is None else analyses.succeeded, - ), - ) - pivots: dict[int, tuple[str, tuple[SchemaDynamicColumnFamily, ...]]] = standalone_pivot_models( - model_inputs=inputs.model_inputs, analysed={} if analyses is None else analyses.positions - ) - proofs: dict[int, ProofRow | None] = dict( - zip( - pivots, - prove_native_dynamic_contracts( - session=session, - tables=NativePivotTables( - dialect=dialect, - column_types_by_table=column_types_by_table, - authoritative_column_types_by_table=authoritative_column_types_by_table, - column_nullability_by_table=column_nullability_by_table, - dynamic_families_by_table=dynamic_families_by_table, - ), - models=tuple(pivots.values()), - ), - strict=True, - ) - ) - return NativeProjectFacts( - models=tuple( - NativeModelFacts( - deps=resources.model_deps[index], - dynamic_contract=proofs.get(index), - syntax_valid=resources.model_syntax_valid[index], - ) - for index in range(len(inputs.model_inputs)) - ), - resources=resources, - ) - - -def expression_source_shapes( - *, expressions: tuple[str, ...], profile: ExpressionInferenceProfile -) -> tuple[dict[str, str] | None, ...]: - """Infer one shape per expression source natively.""" - - return infer_native_expression_source_shapes(expressions=expressions, profile=profile) - - -def analyze_model_sql(request: NativeModelAnalysisRequest) -> NativeModelAnalyses: - """Analyze models natively; their outcomes stay in the finished session.""" - - return analyze_native_model_sql(request=request) diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py deleted file mode 100644 index 19364899f1..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Assemble every SQL test natively; macro-mocked tests' model queries run the user's macros.""" - -from __future__ import annotations - -import decimal -from collections.abc import Callable -from dataclasses import replace - -import sqlbuild._native as _native -from sqlbuild._native import assemble_native_sql_tests -from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompiledModelSqlTestPayload, - CompiledSqlTest, - CompileModelSqlTestInputPayload, - CompileSqlTestInput, -) -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.sql_test_glue.constants import ( - SQL_TEST_DECIMAL_OVERFLOW, - SQL_TEST_INPUT_FAILURE, -) -from sqlbuild.compiler.sql_test_glue.models import NativeSqlTestAssembly - - -def assemble_sql_tests( - *, - inputs: CompileInputsView, - macro_mock_queries: Callable[[CompileSqlTestInput], dict[str, str]], -) -> tuple[CompiledSqlTest, ...]: - """Assemble every SQL test natively, raising errors in Python's assembly order.""" - - native: tuple[NativeSqlTestAssembly, ...] = assemble_native_sql_tests(inputs=inputs) - tests: list[CompiledSqlTest] = [] - for test_input, assembly in zip(inputs.test_inputs, native, strict=True): - if assembly.test is None: - message: str = assembly.failure.message if assembly.failure is not None else "" - if assembly.failure is not None and assembly.failure.kind == SQL_TEST_INPUT_FAILURE: - raise CompileInputError(message) - raise _native.NativeCompilerError(message) - for key, diagnostic in assembly.diagnostics: - report_compile_diagnostic(key=key, diagnostic=diagnostic) - test: CompiledSqlTest = assembly.test - payload: object = test_input.payload - if isinstance(payload, CompileModelSqlTestInputPayload) and payload.macro_mocks: - test = _with_model_queries(test=test, queries=macro_mock_queries(test_input)) - if assembly.failure is not None and assembly.failure.kind == SQL_TEST_DECIMAL_OVERFLOW: - raise decimal.Overflow([decimal.Overflow]) - tests.append(test) - return tuple(tests) - - -def _with_model_queries(*, test: CompiledSqlTest, queries: dict[str, str]) -> CompiledSqlTest: - payload: object = test.payload - if not isinstance(payload, CompiledModelSqlTestPayload): - return test - return replace(test, payload=replace(payload, model_query_overrides=queries)) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/spans.py b/src/sqlbuild/compiler/compile/_helpers/render/spans.py deleted file mode 100644 index 4128bbdcc9..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/render/spans.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Map offsets in rendered SQL back onto the authored text that produced them.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import ExpansionSpan, MappedOffset - - -def map_output_offset(*, offset: int, spans: tuple[ExpansionSpan, ...]) -> MappedOffset: - """Resolve one rendered offset through a single expansion pass.""" - - mapped: int = offset - span: ExpansionSpan - for span in spans: - if offset < span.output_start: - break - if offset < span.output_end: - return MappedOffset(offset=span.source_start, generated=True) - mapped += (span.source_end - span.source_start) - (span.output_end - span.output_start) - return MappedOffset(offset=mapped, generated=False) - - -def map_through_passes( - *, offset: int, passes: tuple[tuple[ExpansionSpan, ...], ...] -) -> MappedOffset: - """Resolve one rendered offset back through every expansion pass in order.""" - - current: int = offset - generated: bool = False - spans: tuple[ExpansionSpan, ...] - for spans in reversed(passes): - resolved: MappedOffset = map_output_offset(offset=current, spans=spans) - current = resolved.offset - generated = generated or resolved.generated - return MappedOffset(offset=current, generated=generated) diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py deleted file mode 100644 index 47743fb45b..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py +++ /dev/null @@ -1,26 +0,0 @@ -"""SQL-native test compile-semantic extraction helpers.""" - -from __future__ import annotations - -from functools import lru_cache - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompileSqlTestCte, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -@lru_cache(maxsize=4096) -def extract_unclassified_sql_test_ctes( - *, sql: str, file_label: str, syntax: SqlLexicalSyntax -) -> tuple[CompileSqlTestCte, ...]: - """Extract raw top-level CTEs before mode-specific classification.""" - - error: str | None - ctes: list[tuple[str, str]] - error, ctes = _native.scope_test_ctes([(sql, file_label)], syntax.native_mapping)[0] - if error is not None: - raise CompileInputError(error) - return tuple(CompileSqlTestCte(name=name, sql_body=body) for name, body in ctes) diff --git a/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py b/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py index 8211248490..a01db8e479 100644 --- a/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py +++ b/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py @@ -2,9 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.compile._helpers.sql_tests.core import ( - extract_unclassified_sql_test_ctes, -) +from sqlbuild._native import extract_sql_test_ctes from sqlbuild.compiler.compile.models import CompileSqlTestCte from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @@ -18,4 +16,4 @@ def extract( ) -> tuple[CompileSqlTestCte, ...]: """Extract top-level SQL-test CTEs without classifying their roles.""" - return extract_unclassified_sql_test_ctes(sql=sql, file_label=file_label, syntax=syntax) + return extract_sql_test_ctes(sql=sql, file_label=file_label, syntax=syntax) diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 9e29853df1..dfc9b660ad 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -50,7 +50,6 @@ "command_output_sinks": "runtime command outputs, not graph resources", "native_session": "native parse state kept for later stages, not a project input", } -SQL_ANALYSIS_OPT_OUT_ENTRY: str = "sql_analysis false" HOOK_DIRECTORY_NAME: str = "hooks" MODEL_DIRECTORY_NAME: str = "models" NOT_NULL_AUDIT_NAME: str = "not_null" diff --git a/src/sqlbuild/compiler/compile/main/__init__.py b/src/sqlbuild/compiler/compile/main/__init__.py deleted file mode 100644 index 279e9ab38a..0000000000 --- a/src/sqlbuild/compiler/compile/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compile public entry modules.""" diff --git a/src/sqlbuild/compiler/compile/main/_infer_fixture_columns.py b/src/sqlbuild/compiler/compile/main/_infer_fixture_columns.py deleted file mode 100644 index b07f9e25de..0000000000 --- a/src/sqlbuild/compiler/compile/main/_infer_fixture_columns.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public fixture-column inference entrypoint.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile._helpers.analysis.fixture_columns import ( - infer_fixture_column_facts as _infer_fixture_column_facts, -) -from sqlbuild.compiler.compile.models import FixtureColumnInference - - -def infer_fixture_column_facts( - *, query_sql: str, inference_profile: ExpressionInferenceProfile -) -> FixtureColumnInference | None: - """Infer fixture columns and identify explicitly untyped NULL projections.""" - - return _infer_fixture_column_facts( - query_sql=query_sql, - inference_profile=inference_profile, - ) diff --git a/src/sqlbuild/compiler/compile/main/map_expanded_offset.py b/src/sqlbuild/compiler/compile/main/map_expanded_offset.py deleted file mode 100644 index 4482858e88..0000000000 --- a/src/sqlbuild/compiler/compile/main/map_expanded_offset.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Resolve an offset in expanded SQL back onto the authored body.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile._helpers.render.spans import map_through_passes -from sqlbuild.compiler.compile.models import ExpansionSpan, MappedOffset - - -def map_expanded_offset( - *, offset: int, passes: tuple[tuple[ExpansionSpan, ...], ...] -) -> MappedOffset: - """Resolve an offset in expanded SQL back onto the authored body.""" - - return map_through_passes(offset=offset, passes=passes) diff --git a/src/sqlbuild/compiler/planner/classes/fixture_column_inferences.py b/src/sqlbuild/compiler/planner/classes/fixture_column_inferences.py index 0bd817703e..16025222bf 100644 --- a/src/sqlbuild/compiler/planner/classes/fixture_column_inferences.py +++ b/src/sqlbuild/compiler/planner/classes/fixture_column_inferences.py @@ -2,8 +2,8 @@ from __future__ import annotations +from sqlbuild._native import infer_fixture_column_facts from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.main._infer_fixture_columns import infer_fixture_column_facts from sqlbuild.compiler.compile.models import FixtureColumnInference diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 807be35a12..70f7e9f404 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -33,10 +33,7 @@ import sqlbuild.adapter.type_system._helpers.type_normalization as type_normalization import sqlbuild.cli.compile_reuse._helpers.native_reuse as native_reuse import sqlbuild.cli.compile_reuse._helpers.provider_settings as provider_settings -import sqlbuild.compiler.compile._helpers.diagnostics.recovery as diagnostic_recovery import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module -import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stages -import sqlbuild.compiler.compile._helpers.native_stages.sql_tests as native_sql_test_stage import sqlbuild.compiler.frontier.main.compiled_code_identity as compiled_code_identity_module import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class from scripts.cold_compile_performance.main.read_compile_measurement import read_compile_measurement @@ -4811,11 +4808,11 @@ def full_macro_call_runs( _ANALYSIS_SEAMS: tuple[tuple[ModuleType, str], ...] = ( - (native_stages, "assemble_native_project_resources"), - (native_stages, "infer_native_expression_source_shapes"), - (native_stages, "analyze_native_model_sql"), - (native_sql_test_stage, "assemble_native_sql_tests"), - (diagnostic_recovery, "complete_semantic_project"), + (native_module, "assemble_native_project_resources"), + (native_module, "infer_native_expression_source_shapes"), + (native_module, "analyze_native_model_sql"), + (native_module, "assemble_native_sql_tests"), + (native_module, "complete_semantic_project"), (native_module, "evaluate_model_contracts"), (native_module, "promotion_conflict_diagnostics"), (native_module, "build_project_column_lineage"), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index 06c9e50f03..ab8d8c437d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -30,7 +30,7 @@ description="models_contracts_lineage_and_sql_tests", expected_native_returns={ "assemble_native_project_resources": [ANY], - "infer_native_expression_source_shapes": [ANY], + "infer_native_expression_source_shapes": [ANY, ANY, ANY, ANY], "analyze_native_model_sql": [ANY], "assemble_native_sql_tests": [ANY], "complete_semantic_project": [ANY], diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 5341e0102e..cc5bc10b8b 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -14,8 +14,6 @@ import pytest import sqlbuild._native as native_module -import sqlbuild.compiler.compile._helpers.diagnostics.recovery as semantic_recovery -import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly from sqlbuild._native import ( analyze_native_model_sql, build_compile_inputs, @@ -418,7 +416,7 @@ def native_pivot_proofs(*, monkeypatch: pytest.MonkeyPatch) -> NativePivotProofs recorded_proofs: NativePivotProofs = NativePivotProofs() native_proofs: Callable[..., tuple[ProofRow | None, ...]] = ( - native_stage_assembly.prove_native_dynamic_contracts + native_module.prove_native_dynamic_contracts ) def recorded(**keywords: Any) -> tuple[ProofRow | None, ...]: @@ -428,7 +426,7 @@ def recorded(**keywords: Any) -> tuple[ProofRow | None, ...]: recorded_proofs.session_proofs += proven * (keywords["session"] is not None) return proofs - monkeypatch.setattr(native_stage_assembly, "prove_native_dynamic_contracts", recorded) + monkeypatch.setattr(native_module, "prove_native_dynamic_contracts", recorded) return recorded_proofs @@ -587,7 +585,7 @@ def analyse_natively( `runs.golden` gathers the entries the Python oracle recorded for the same seam calls. """ - original_analysis: Callable[..., NativeModelAnalyses] = native_stage_assembly.analyze_model_sql + original_analysis: Callable[..., NativeModelAnalyses] = analyze_native_model_sql def analysed_three_ways(request: NativeModelAnalysisRequest) -> NativeModelAnalyses: profile: ExpressionInferenceProfile = request.inference_profile @@ -607,7 +605,9 @@ def analysed_three_ways(request: NativeModelAnalysisRequest) -> NativeModelAnaly cached_views.append( (_analysis_views(_projected(cached, request, runs)), _catalog_view(catalog)) ) - native: NativeModelAnalyses = original_analysis(replace(request, analysis_cache=None)) + native: NativeModelAnalyses = original_analysis( + request=replace(request, analysis_cache=None) + ) session: Any = native.session analyses: dict[str, ModelSqlAnalysis] = _projected(native, request, runs) uncached_view: object = (_analysis_views(analyses), _catalog_view(profile.binding_catalog)) @@ -634,18 +634,24 @@ def recorded_shapes( runs.golden.append(golden_entry("shapes", (native, catalog.expression_shapes))) return native - completed: Callable[..., CompiledProjectView] = semantic_recovery.complete_semantic_diagnostics + completed: Callable[..., CompiledProjectView] = native_module.complete_semantic_project def recorded_proofs(*, project: CompiledProjectView, **keywords: Any) -> CompiledProjectView: runs.golden.extend( golden_entry("proof", model.dynamic_column_contract) for model in project.models ) - return completed(project=project, **keywords) + with monkeypatch.context() as completing: + completing.setattr( + native_module, + "infer_native_expression_source_shapes", + infer_native_expression_source_shapes, + ) + return completed(project=project, **keywords) with monkeypatch.context() as patch: - patch.setattr(semantic_recovery, "complete_semantic_diagnostics", recorded_proofs) - patch.setattr(native_stage_assembly, "analyze_model_sql", analysed_three_ways) - patch.setattr(native_stage_assembly, "expression_source_shapes", recorded_shapes) + patch.setattr(native_module, "complete_semantic_project", recorded_proofs) + patch.setattr(native_module, "analyze_native_model_sql", analysed_three_ways) + patch.setattr(native_module, "infer_native_expression_source_shapes", recorded_shapes) with suppress(CompileInputError): _ = native_module.assemble_project( inputs=inputs, @@ -691,7 +697,7 @@ def captured(request: NativeModelAnalysisRequest) -> NativeModelAnalyses: monkeypatch.context() as patch, suppress(CompileInputError, native_module.NativeCompilerError), ): - patch.setattr(native_stage_assembly, "analyze_model_sql", captured) + patch.setattr(native_module, "analyze_native_model_sql", captured) _ = native_module.assemble_project( inputs=inputs, inference_profile=inference_profile, diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py index 82852ea94d..2bf800b5c8 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py @@ -12,7 +12,6 @@ import pytest import sqlbuild._native as native_module -import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly from sqlbuild._native import infer_native_query_columns from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeModelAnalyses @@ -130,9 +129,9 @@ def test_given_session_without_model_analysis_when_assembling_then_raises_native test_case: MissingSessionAnalysisTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr( - native_stage_assembly, - "analyze_model_sql", - lambda _request: NativeModelAnalyses(positions={}, succeeded=frozenset(), session=None), + native_module, + "analyze_native_model_sql", + lambda request: NativeModelAnalyses(positions={}, succeeded=frozenset(), session=None), ) with pytest.raises(native_module.NativeCompilerError) as raised: diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index f3fd696ec9..33b498ca3b 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -9,7 +9,6 @@ import pytest import sqlbuild._native as native_module -import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly from sqlbuild._native import ( build_compile_inputs, build_compiled_project, @@ -218,13 +217,12 @@ def traced_shape_batches(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, ... """Record each shape request's expressions the binding catalog's shape cache misses.""" batches: list[tuple[str, ...]] = [] - for module in (native_stage_assembly, native_module): - original: Callable[..., Any] = module.infer_native_expression_source_shapes - monkeypatch.setattr( - module, - "infer_native_expression_source_shapes", - functools.partial(_traced_shapes, original=original, batches=batches), - ) + original: Callable[..., Any] = native_module.infer_native_expression_source_shapes + monkeypatch.setattr( + native_module, + "infer_native_expression_source_shapes", + functools.partial(_traced_shapes, original=original, batches=batches), + ) return batches diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py index b19e6a429c..8e58afe6ad 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py @@ -6,6 +6,7 @@ import pytest +import sqlbuild.compiler.sql_analysis.models as sql_analysis_models from sqlbuild._native import ( build_compiled_project, discover_project_inputs, @@ -14,7 +15,6 @@ from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile._helpers.analysis import binding_requests from sqlbuild.compiler.compile.models import CompiledObjectKey, CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( @@ -180,7 +180,7 @@ def test_given_sql_when_case_policy_enabled_then_fused_validation_is_reused( def repeated_binding(**kwargs: Any) -> None: raise AssertionError("already validated SQL was normalized and bound again") - monkeypatch.setattr(binding_requests, "get_complete_schema_binding_request", repeated_binding) + monkeypatch.setattr(sql_analysis_models, "SqlSchemaValidationRequest", repeated_binding) assert ( main( [ diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index c17d00de94..d81bc4f55a 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -13,14 +13,10 @@ import pytest import sqlbuild._native as native_module -import sqlbuild.compiler.compile._helpers.diagnostics.recovery as semantic_recovery from sqlbuild._native import build_project_graph, discover_project_inputs, project_with from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.analysis_session.types import DiagnosticRow -from sqlbuild.compiler.compile._helpers.diagnostics.recovery import ( - complete_semantic_diagnostics, -) from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.integration.src.sqlbuild.compiler.analysis_session.helpers import ( @@ -84,13 +80,13 @@ @dataclass(frozen=True) class SemanticInputs: - """The arguments the Python compile passed to `complete_semantic_diagnostics`.""" + """The arguments the compile passed to `complete_semantic_project`.""" project: CompiledProjectView profile: ExpressionInferenceProfile binding_results: dict[str, list[DiagnosticRow]] resource_sql_analysis: bool - native_session: Any | None = None + session: Any | None = None def _typo(rng: random.Random, column: str) -> str: @@ -196,12 +192,14 @@ def captured_semantic_inputs( _ = path.write_text(contents, encoding="utf-8") captured: list[SemanticInputs] = [] + complete: Callable[..., CompiledProjectView] = native_module.complete_semantic_project + def capture(**arguments: Any) -> CompiledProjectView: captured.append(SemanticInputs(**arguments)) - return complete_semantic_diagnostics(**arguments) + return complete(**arguments) with monkeypatch.context() as patch: - patch.setattr(semantic_recovery, "complete_semantic_diagnostics", capture) + patch.setattr(native_module, "complete_semantic_project", capture) _ = build_project_graph( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), @@ -243,7 +241,7 @@ def native_completion(inputs: SemanticInputs) -> CompiledProjectView: profile=inputs.profile, binding_results=inputs.binding_results, resource_sql_analysis=inputs.resource_sql_analysis, - session=inputs.native_session, + session=inputs.session, ) @@ -271,13 +269,13 @@ def proven_output_count(project: CompiledProjectView) -> int: def without_session(inputs: SemanticInputs) -> SemanticInputs: """The same stage inputs with every model's output names and lineage in the payload.""" - return replace(inputs, native_session=None) + return replace(inputs, session=None) def session_fact_models(inputs: SemanticInputs) -> frozenset[str]: """Models whose output names and lineage the finished session supplies to the stage.""" - kept: frozenset[str] = frozenset(getattr(inputs.native_session, "fact_models", ())) + kept: frozenset[str] = frozenset(getattr(inputs.session, "fact_models", ())) proven: frozenset[str] = frozenset( model.name for model in filter(_output_proven, inputs.project.models) ) diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py index 1ec117546b..37a7d81982 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py @@ -285,7 +285,7 @@ def test_given_compiles_when_completing_from_the_session_then_matches_payload_an session_models: int = sum(len(selected) for selected in selections) model_count: int = sum(len(inputs.project.models) for inputs in captured) - assert all(inputs.native_session is not None for inputs in captured) + assert all(inputs.session is not None for inputs in captured) assert ( golden_differences( read_golden(golden_name("semantic_session", test_case.description)), golden diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py index 94547ebe5c..0332ad8888 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py @@ -12,9 +12,9 @@ import pytest +import sqlbuild._native as native_module from sqlbuild._native import build_project_graph, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile._helpers.native_stages import sql_tests as sql_test_stage from sqlbuild.compiler.compile.models import ( CompiledSqlTest, ) @@ -463,9 +463,7 @@ def record_native_assemblies(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str] """ answers: Counter[str] = Counter() - seam: Callable[..., tuple[NativeSqlTestAssembly, ...]] = ( - sql_test_stage.assemble_native_sql_tests - ) + seam: Callable[..., tuple[NativeSqlTestAssembly, ...]] = native_module.assemble_native_sql_tests def counted_assemblies(*, inputs: CompileInputsView) -> tuple[NativeSqlTestAssembly, ...]: assemblies: tuple[NativeSqlTestAssembly, ...] = seam(inputs=inputs) @@ -480,7 +478,7 @@ def counted_assemblies(*, inputs: CompileInputsView) -> tuple[NativeSqlTestAssem ) return assemblies - monkeypatch.setattr(sql_test_stage, "assemble_native_sql_tests", counted_assemblies) + monkeypatch.setattr(native_module, "assemble_native_sql_tests", counted_assemblies) return answers diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py index cd7a18324b..fbe30bb55e 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py @@ -14,9 +14,7 @@ ) from tests.unit.src.sqlbuild.cli.compile_reuse.main.helpers import compile_in_fresh_process -_FULL_COMPILE_MODULES: tuple[str, ...] = ( - "sqlbuild.compiler.compile._helpers.native_stages.assembly", -) +_FULL_COMPILE_MODULES: tuple[str, ...] = ("sqlbuild.compiler.analysis_session.models",) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py index d8b5f93900..88f9f0f96c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import expand_authored_sql_with_spans -from sqlbuild.compiler.compile._helpers.render.spans import map_through_passes +from sqlbuild._native import expand_authored_sql_with_spans, map_expanded_offset from sqlbuild.compiler.compile.models import ExpansionSpan, LoadedMacro, MacroContext, MappedOffset from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ExpandWithSpansTestCase from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( @@ -98,7 +97,7 @@ def test_given_expanded_offset_when_mapping_then_authored_position_matches( collection_rendering=DUCKDB_COMPILE_ADAPTER_CONTEXT.collection_rendering, ) assert expanded_sql == test_case.expected_expanded_sql - resolved: MappedOffset = map_through_passes( + resolved: MappedOffset = map_expanded_offset( offset=test_case.probe_expanded_offset, passes=passes ) assert resolved.offset == test_case.expected_authored_offset diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_spans.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_spans.py index 29ad65be0c..359f824121 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_spans.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_spans.py @@ -4,10 +4,7 @@ import pytest -from sqlbuild.compiler.compile._helpers.render.spans import ( - map_output_offset, - map_through_passes, -) +from sqlbuild._native import map_expanded_offset from sqlbuild.compiler.compile.models import ExpansionSpan, MappedOffset from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( MapOutputOffsetTestCase, @@ -76,7 +73,7 @@ def test_given_rendered_offset_when_mapping_one_pass_then_source_offset_matches( test_case: MapOutputOffsetTestCase, ) -> None: - resolved: MappedOffset = map_output_offset(offset=test_case.offset, spans=test_case.spans) + resolved: MappedOffset = map_expanded_offset(offset=test_case.offset, passes=(test_case.spans,)) assert resolved.offset == test_case.expected_offset assert resolved.generated == test_case.expected_generated @@ -137,6 +134,6 @@ def test_given_rendered_offset_when_mapping_one_pass_then_source_offset_matches( def test_given_rendered_offset_when_mapping_all_passes_then_authored_offset_matches( test_case: MapThroughPassesTestCase, ) -> None: - resolved: MappedOffset = map_through_passes(offset=test_case.offset, passes=test_case.passes) + resolved: MappedOffset = map_expanded_offset(offset=test_case.offset, passes=test_case.passes) assert resolved.offset == test_case.expected_offset assert resolved.generated == test_case.expected_generated diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py index d729ea72fb..07f3824c8f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py @@ -2,12 +2,9 @@ import pytest -from sqlbuild._native import infer_native_query_columns +from sqlbuild._native import infer_native_query_columns, substitute_placeholder_defaults from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns -from sqlbuild.compiler.compile._helpers.analysis.reference_names import ( - substitute_placeholder_defaults, -) from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, CompiledLineageSourceFact, @@ -944,7 +941,7 @@ def test_given_sql_with_placeholders_when_substituting_then_returns_expected( test_case: SubstitutePlaceholderDefaultsTestCase, ) -> None: result: str = substitute_placeholder_defaults( - query_sql=test_case.query_sql, placeholders=test_case.placeholders + test_case.query_sql, list(test_case.placeholders.items()) ) assert result == test_case.expected_sql diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py index 016fc0b8cd..6703d65bae 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py @@ -11,8 +11,8 @@ import sqlbuild.compiler.planner.classes.fixture_column_inferences as fixture_inferences import sqlbuild.compiler.planner.classes.migration_fingerprint_cache as fingerprint_cache +from sqlbuild._native import infer_fixture_column_facts from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.main._infer_fixture_columns import infer_fixture_column_facts from sqlbuild.compiler.compile.models import FixtureColumnInference from sqlbuild.compiler.planner._helpers.migrations.fingerprint import build_migration_fingerprint diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_fixture_column_inferences.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_fixture_column_inferences.py index b474b847d8..a731ac9d73 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_fixture_column_inferences.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_fixture_column_inferences.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.compile.main._infer_fixture_columns import infer_fixture_column_facts +from sqlbuild._native import infer_fixture_column_facts from sqlbuild.compiler.compile.models import FixtureColumnInference from sqlbuild.compiler.planner.classes.fixture_column_inferences import FixtureColumnInferences from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import ( diff --git a/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py b/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py index 4f1ef80581..a2dd5fd97d 100644 --- a/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py +++ b/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py @@ -6,8 +6,11 @@ from scripts.setting_help.main.find_settings_without_snippet import find_settings_without_snippet from scripts.setting_help.main.parse_setting_snippet import parse_setting_snippet -from sqlbuild._native import explicit_references_help, microbatch_guidance -from sqlbuild.compiler.compile._helpers.analysis.validation import sql_analysis_opt_out_help +from sqlbuild._native import ( + explicit_references_help, + microbatch_guidance, + sql_analysis_opt_out_help, +) from sqlbuild.errors.setting_help.main.join_helps import join_helps from sqlbuild.errors.setting_help.main.model_header_help import model_header_help from sqlbuild.errors.setting_help.main.setting_help import setting_help diff --git a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py index 7fabbf9cad..8286b4a231 100644 --- a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py @@ -7,8 +7,7 @@ import pytest -from sqlbuild._native import expand_sql_with_spans, run_lint -from sqlbuild.compiler.compile.main.map_expanded_offset import map_expanded_offset +from sqlbuild._native import expand_sql_with_spans, map_expanded_offset, run_lint from sqlbuild.compiler.compile.models import MappedOffset, SqlExpansionContext from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.lint._helpers import expansion From c1cbaedf3f69ae8ea3feda1d3e4513ed391a0441 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:27:29 +0100 Subject: [PATCH 34/40] refactor: move the SQL value, reference, manifest and lint helper wrappers into native bindings --- .../_helpers/build_command/planning.rs | 3 +- .../_helpers/compile_attachment/mod.rs | 2 +- .../_helpers/compile_driver/phases.rs | 4 +- .../src/bindings/_helpers/contracts/mod.rs | 1 + .../_helpers/contracts/registration.rs | 5 +- .../_helpers/contracts/resource_identities.rs | 52 +++++- .../bindings/_helpers/contracts/sql_values.rs | 31 ++-- .../_helpers/contracts/state_literals.rs | 158 ++++++++++++++++++ .../bindings/_helpers/format_command/files.rs | 14 +- .../_helpers/format_command/registration.rs | 7 +- .../src/bindings/_helpers/graph/views.rs | 2 +- .../src/bindings/_helpers/manifest/builder.rs | 74 ++++---- .../bindings/_helpers/rules/lint_expansion.rs | 128 ++++++++++++++ .../src/bindings/_helpers/rules/mod.rs | 1 + .../bindings/_helpers/rules/registration.rs | 4 +- .../bindings/_helpers/rules/relation_keys.rs | 69 +++++++- .../src/bindings/_helpers/rules/run_lint.rs | 20 ++- fensu.toml | 15 ++ src/sqlbuild/_native.pyi | 55 +++--- .../state_sql/_helpers/source_freshness.py | 2 +- .../compile/_helpers/render/user_macros.py | 4 +- .../compiler/contract_adoption/main/_write.py | 2 +- .../discovery/_helpers/filesystem/core.py | 4 +- src/sqlbuild/compiler/manifest/__init__.py | 0 .../compiler/manifest/main/__init__.py | 0 .../manifest/main/_build_graph_manifest.py | 33 ---- src/sqlbuild/compiler/manifest/main/build.py | 33 ---- .../compiler/planner/_helpers/resolve/refs.py | 25 ++- .../planner/_helpers/resolve/sources.py | 7 +- .../planner/_helpers/scenario/relations.py | 23 +-- .../_helpers/warehouse/source_deferral.py | 6 +- .../compiler/references/_helpers/__init__.py | 1 - .../compiler/references/_helpers/patterns.py | 19 --- src/sqlbuild/compiler/references/constants.py | 18 +- .../compiler/references/main/__init__.py | 1 - .../main/_quoted_reference_call_pattern.py | 14 -- .../_reference_call_prefix_pattern_text.py | 12 -- .../compiler/resource_names/main/__init__.py | 1 - .../main/_validate_resource_identity.py | 23 --- .../resource_names/main/function_node_type.py | 17 -- .../executor/audit_results/_helpers/sql.py | 2 +- .../executor/functions/main/_execute.py | 2 +- .../executor/janitor_events/_helpers/sql.py | 2 +- .../executor/node_results/_helpers/sql.py | 2 +- src/sqlbuild/lint/_helpers/expansion.py | 60 ------- src/sqlbuild/lint/_helpers/fixes.py | 19 +-- .../lint/main/build_expansion_context.py | 22 --- .../lint/main/compiled_relation_keys.py | 17 -- .../lint/main/declared_relation_keys.py | 32 ---- src/sqlbuild/lint/main/write_atomically.py | 13 -- src/sqlbuild/microbatches/_helpers/sql.py | 2 +- .../rule_engine/_helpers/run/rule_threads.py | 5 +- .../rule_engine/_helpers/run/rules.py | 8 +- src/sqlbuild/sql_values/_helpers/__init__.py | 0 .../sql_values/_helpers/state_literal.py | 71 -------- src/sqlbuild/sql_values/main/__init__.py | 0 src/sqlbuild/sql_values/main/normalize.py | 26 --- .../sql_values/main/render_state_literal.py | 12 -- .../sql_values/main/validate_rendered_size.py | 19 --- .../duckdb/test_typed_sql_rendering.py | 2 +- .../main/test_compile_startup_imports.py | 2 - .../src/sqlbuild/compiler/pipeline/helpers.py | 2 +- .../contract/typed_sql_rendering/helpers.py | 2 +- .../compile/test_compile_target_writer.py | 2 +- .../compiler/compile/_helpers/helpers.py | 2 +- .../_helpers/test_resource_identities.py | 5 +- .../src/sqlbuild/compiler/manifest/helpers.py | 2 +- .../sqlbuild/compiler/manifest/test_main.py | 2 +- .../scopes/test_builder_and_visibility.py | 2 +- .../src/sqlbuild/lint/test_expanded_lint.py | 15 +- .../_helpers/engine/test_typed_constants.py | 2 +- .../src/sqlbuild/sql_values/_test_types.py | 8 - .../sqlbuild/sql_values/test_normalization.py | 30 +--- .../sqlbuild/sql_values/test_state_literal.py | 2 +- 74 files changed, 623 insertions(+), 661 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs delete mode 100644 src/sqlbuild/compiler/manifest/__init__.py delete mode 100644 src/sqlbuild/compiler/manifest/main/__init__.py delete mode 100644 src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py delete mode 100644 src/sqlbuild/compiler/manifest/main/build.py delete mode 100644 src/sqlbuild/compiler/references/_helpers/__init__.py delete mode 100644 src/sqlbuild/compiler/references/_helpers/patterns.py delete mode 100644 src/sqlbuild/compiler/references/main/__init__.py delete mode 100644 src/sqlbuild/compiler/references/main/_quoted_reference_call_pattern.py delete mode 100644 src/sqlbuild/compiler/references/main/_reference_call_prefix_pattern_text.py delete mode 100644 src/sqlbuild/compiler/resource_names/main/__init__.py delete mode 100644 src/sqlbuild/compiler/resource_names/main/_validate_resource_identity.py delete mode 100644 src/sqlbuild/compiler/resource_names/main/function_node_type.py delete mode 100644 src/sqlbuild/lint/main/build_expansion_context.py delete mode 100644 src/sqlbuild/lint/main/compiled_relation_keys.py delete mode 100644 src/sqlbuild/lint/main/declared_relation_keys.py delete mode 100644 src/sqlbuild/lint/main/write_atomically.py delete mode 100644 src/sqlbuild/sql_values/_helpers/__init__.py delete mode 100644 src/sqlbuild/sql_values/_helpers/state_literal.py delete mode 100644 src/sqlbuild/sql_values/main/__init__.py delete mode 100644 src/sqlbuild/sql_values/main/normalize.py delete mode 100644 src/sqlbuild/sql_values/main/render_state_literal.py delete mode 100644 src/sqlbuild/sql_values/main/validate_rendered_size.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index 9bd9b0908c..02bcdfc731 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -17,7 +17,6 @@ const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.exte const VERBOSE_INSPECTION_MODULE: &str = "sqlbuild.cli.progress.main.verbose_inspection_output"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; -const MANIFEST_MODULE: &str = "sqlbuild.compiler.manifest.main.build"; const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; /// Clone out-of-selection boundaries from the defer-clone origin before planning. @@ -212,7 +211,7 @@ pub(crate) fn write_compile_target( let plan: Bound<'_, PyAny> = pipeline.getattr("plan_output")?; let manifest: Bound<'_, PyAny> = if context.request_flag("manifest")? { call_with( - &imported(py, MANIFEST_MODULE, "build_manifest")?, + &imported(py, NATIVE_MODULE, "build_manifest")?, &[ ("project", pipeline.getattr("project")?), ("plan_output", plan.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs index f9cea53494..5737802a12 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs @@ -4,6 +4,6 @@ pub(crate) mod effective_config; pub(crate) mod model_config; pub(crate) mod model_inputs; pub(crate) mod registration; -mod sql_expansion_context; +pub(crate) mod sql_expansion_context; mod storage_policies; pub(crate) mod target; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index 6cec2848d6..5f8f1e40c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -12,7 +12,7 @@ use crate::bindings::_helpers::compile_command::target_writer::{ use crate::bindings::_helpers::compile_driver::python_calls::{Status, call_with, imported}; use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; -const MANIFEST_MODULE: &str = "sqlbuild.compiler.manifest.main._build_graph_manifest"; +const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const PYTHON_NODES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild.cli.output.models"; @@ -68,7 +68,7 @@ pub(crate) fn manifest_payload<'py>( ], )?; let payload: Bound<'py, PyAny> = call_with( - &imported(py, MANIFEST_MODULE, "build_graph_manifest")?, + &imported(py, NATIVE_MODULE, "build_manifest")?, &[ ("project", graph.getattr("project")?), ("plan_output", py.None().into_bound(py)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs index 4eadc39c96..bd7003c946 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs @@ -6,3 +6,4 @@ pub(crate) mod registration; pub(crate) mod resource_identities; pub(crate) mod spec_resolution; pub(crate) mod sql_values; +pub(crate) mod state_literals; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs index 6d04c9a5cf..cc4f591a2b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::contracts::{ contract_diagnostics, effective_config, project_config, resource_identities, spec_resolution, - sql_values, + sql_values, state_literals, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -13,5 +13,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { project_config::register(module)?; resource_identities::register(module)?; spec_resolution::register(module)?; - sql_values::register(module) + sql_values::register(module)?; + state_literals::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs index b890e7365c..7eb4fc6175 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs @@ -1,20 +1,56 @@ -//! Canonical snake_case resource identity checks. +//! Canonical snake_case resource identity checks and SQL function node types. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyString, PyStringMethods}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyString, PyStringMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::resource_identities::main::resource_identity_failure::resource_identity_failure; -/// The snake_case suggestion and help for a non-canonical identity, or `None`. +const RESOURCE_NAME_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.resource_names.exceptions"; +const TABLE_FUNCTION_NODE_TYPE: &str = "table_fn"; +const UDF_NODE_TYPE: &str = "udf"; + +/// Reject one identity unless it uses canonical lowercase snake_case. #[pyfunction] -fn resource_identity_suggestion( +#[pyo3(signature = (*, name, kind, path, private_identity = false))] +fn validate_resource_identity( name: &Bound<'_, PyString>, + kind: &str, + path: &Bound<'_, PyAny>, private_identity: bool, -) -> PyResult> { +) -> Result<(), PyErr> { + let py: Python<'_> = name.py(); let text = name.to_string_lossy(); - Ok(resource_identity_failure(&text, private_identity)) + let Some((corrected_name, help_text)) = resource_identity_failure(&text, private_identity) + else { + return Ok(()); + }; + let message: String = format!( + "Invalid {kind} identity '{text}' in {}; use snake_case '{corrected_name}'", + path.str()? + ); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("help", help_text)?; + let error: Bound<'_, PyAny> = py + .import(RESOURCE_NAME_EXCEPTIONS_MODULE)? + .getattr("ResourceIdentityError")? + .call((message,), Some(&keywords))?; + Err(PyErr::from_value(error)) +} + +/// The public node type of a SQL function: a table function when it returns columns. +#[pyfunction] +#[pyo3(signature = (*, return_columns))] +fn function_node_type(return_columns: &Bound<'_, PyAny>) -> PyResult<&'static str> { + Ok(if return_columns.is_truthy()? { + TABLE_FUNCTION_NODE_TYPE + } else { + UDF_NODE_TYPE + }) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resource_identity_suggestion, module)?) + module.add_function(wrap_pyfunction!(validate_resource_identity, module)?)?; + module.add_function(wrap_pyfunction!(function_node_type, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/sql_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/sql_values.rs index c289f13901..aea1abc5b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/sql_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/sql_values.rs @@ -10,7 +10,7 @@ use pyo3::types::{ use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_core::models::{SqlLogicalType, SqlValue}; use sqlbuild_model_config::sql_values::errors::SqlValueFailure; -use sqlbuild_model_config::sql_values::main::normalize_sql_value::normalize_sql_value; +use sqlbuild_model_config::sql_values::main::normalize_sql_value::normalize_sql_value as normalized; use sqlbuild_model_config::sql_values::models::{RawKey, RawSqlValue, SqlValueLimits}; const MODELS_MODULE: &str = "sqlbuild.sql_values.models"; @@ -191,27 +191,34 @@ pub(crate) fn sql_value_error(py: Python<'_>, failure: SqlValueFailure) -> PyRes /// `normalize_sql_value`: the canonical `SqlValue` of an authored value. #[pyfunction] -fn normalize_typed_sql_value<'py>( - py: Python<'py>, +#[pyo3(signature = (*, raw_value, context, explicit_type = None, limits = None))] +fn normalize_sql_value<'py>( raw_value: &Bound<'py, PyAny>, - typing: (String, Option), - limits: (i64, i64, i64), + context: &str, + explicit_type: Option<&str>, + limits: Option<&Bound<'py, PyAny>>, ) -> PyResult> { - let (context, explicit_type) = typing; + let py: Python<'py> = raw_value.py(); let classes = SqlValueClasses::load(py)?; let raw = self::raw_value(raw_value, &classes)?; - let (max_depth, max_elements, max_size) = limits; + let applied: Bound<'py, PyAny> = match limits { + Some(limits) if !limits.is_none() => limits.clone(), + _ => py + .import(MODELS_MODULE)? + .getattr("SqlValueLimits")? + .call0()?, + }; let limits = SqlValueLimits { - max_depth, - max_elements, - max_size, + max_depth: applied.getattr("max_depth")?.extract()?, + max_elements: applied.getattr("max_elements")?.extract()?, + max_size: applied.getattr("max_size")?.extract()?, }; - match normalize_sql_value(&raw, &context, explicit_type.as_deref(), limits) { + match normalized(&raw, context, explicit_type, limits) { Ok(value) => sql_value(&value, &classes), Err(failure) => Err(sql_value_error(py, failure)?), } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(normalize_typed_sql_value, module)?) + module.add_function(wrap_pyfunction!(normalize_sql_value, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs new file mode 100644 index 0000000000..58635c12f9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs @@ -0,0 +1,158 @@ +//! Portable typed SQL literals for warehouse-state columns. + +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyInt, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; + +const EXCEPTIONS_MODULE: &str = "sqlbuild.sql_values.exceptions"; +const DATETIME_MODULE: &str = "datetime"; +const JSON_MODULE: &str = "json"; +const STRING_TYPE: &str = "string"; +const INTEGER_TYPE: &str = "integer"; +const BOOLEAN_TYPE: &str = "boolean"; +const TIMESTAMP_TYPE: &str = "timestamp"; +const TEXT_TIMESTAMP_TYPE: &str = "text_timestamp"; +const DATE_TYPE: &str = "date"; +const JSON_TYPE: &str = "json"; + +/// `render_state_sql_literal(*, value, declared_type)`: `value` as its state column's literal. +#[pyfunction] +#[pyo3(signature = (*, value, declared_type))] +fn render_state_sql_literal( + py: Python<'_>, + value: &Bound<'_, PyAny>, + declared_type: &str, +) -> PyResult { + if value.is_none() { + return Ok(typed_null(declared_type).to_owned()); + } + match declared_type { + STRING_TYPE => { + if !value.is_instance_of::() { + return Err(rendering_error( + py, + "state string columns require str values", + )); + } + Ok(quoted(&value.extract::()?)) + } + INTEGER_TYPE => { + if value.is_instance_of::() || !value.is_instance_of::() { + return Err(rendering_error( + py, + "state integer columns require int values", + )); + } + let rendered: String = value.call_method1("__format__", ("",))?.extract()?; + Ok(format!("CAST({rendered} AS BIGINT)")) + } + BOOLEAN_TYPE => { + if !value.is_instance_of::() { + return Err(rendering_error( + py, + "state boolean columns require bool values", + )); + } + Ok(if value.is_truthy()? { "TRUE" } else { "FALSE" }.to_owned()) + } + TIMESTAMP_TYPE | TEXT_TIMESTAMP_TYPE => timestamp_literal(value, declared_type), + DATE_TYPE => { + let datetime = py.import(DATETIME_MODULE)?; + if value.is_instance(&datetime.getattr("datetime")?)? + || !value.is_instance(&datetime.getattr("date")?)? + { + return Err(rendering_error( + py, + "state date columns require date values", + )); + } + let iso: String = value.call_method0("isoformat")?.extract()?; + Ok(format!("CAST({} AS DATE)", quoted(&iso))) + } + JSON_TYPE => json_literal(value), + other => Err(PyValueError::new_err(format!( + "unknown state SQL value type {other:?}" + ))), + } +} + +fn timestamp_literal(value: &Bound<'_, PyAny>, declared_type: &str) -> PyResult { + let py: Python<'_> = value.py(); + let datetime = py.import(DATETIME_MODULE)?; + if !value.is_instance(&datetime.getattr("datetime")?)? { + return Err(rendering_error( + py, + "state timestamp columns require datetime values", + )); + } + let utc: Bound<'_, PyAny> = datetime.getattr("UTC")?; + let normalized: Bound<'_, PyAny> = + if value.getattr("tzinfo")?.is_none() || value.call_method0("utcoffset")?.is_none() { + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("tzinfo", &utc)?; + value.call_method("replace", (), Some(&arguments))? + } else { + value.call_method1("astimezone", (&utc,))? + }; + let rendered: String = quoted(&normalized.call_method0("isoformat")?.extract::()?); + if declared_type == TEXT_TIMESTAMP_TYPE { + return Ok(rendered); + } + Ok(format!("CAST({rendered} AS TIMESTAMP)")) +} + +fn json_literal(value: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = value.py(); + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("ensure_ascii", false)?; + arguments.set_item("separators", (",", ":"))?; + arguments.set_item("sort_keys", true)?; + match py + .import(JSON_MODULE)? + .getattr("dumps")? + .call((value,), Some(&arguments)) + { + Ok(serialized) => Ok(quoted(&serialized.extract::()?)), + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + let raised: PyErr = + rendering_error(py, "state JSON columns require JSON-serializable values"); + raised.set_cause(py, Some(error)); + Err(raised) + } + Err(error) => Err(error), + } +} + +fn quoted(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +fn typed_null(declared_type: &str) -> &'static str { + match declared_type { + INTEGER_TYPE => "CAST(NULL AS BIGINT)", + TIMESTAMP_TYPE => "CAST(NULL AS TIMESTAMP)", + DATE_TYPE => "CAST(NULL AS DATE)", + _ => "NULL", + } +} + +fn rendering_error(py: Python<'_>, message: &str) -> PyErr { + let raised = || -> PyResult { + let error: Bound<'_, PyAny> = py + .import(EXCEPTIONS_MODULE)? + .getattr("SqlValueRenderingError")? + .call1((message,))?; + Ok(PyErr::from_value(error)) + }; + raised().unwrap_or_else(|error| error) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_state_sql_literal, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs index 85bc4e2fc9..7f607c3774 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs @@ -6,8 +6,9 @@ use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; use pyo3::exceptions::PyOSError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; const LINE_FEED: &str = "\n"; const CARRIAGE_RETURN_LINE_FEED: &str = "\r\n"; @@ -87,6 +88,17 @@ fn write_atomically(path: &Path, contents: &str) -> PyResult<()> { written.map_err(|error| PyOSError::new_err(format!("{}: {error}", path.display()))) } +/// `write_atomically(*, path, contents)`: replace a text file atomically, keeping its mode. +#[pyfunction] +#[pyo3(name = "write_atomically", signature = (*, path, contents))] +fn write_file_atomically(path: PathBuf, contents: &str) -> PyResult<()> { + write_atomically(&path, contents) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(write_file_atomically, module)?) +} + /// The changed files in path order, written in place when `write` is set. pub(crate) fn persist_changes<'py>( (files, updated): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/registration.rs index c2bf6cb02c..1b00100a44 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/registration.rs @@ -1,9 +1,10 @@ -//! Register the native `sqb format` command. +//! Register the native `sqb format` command and its atomic file writer. use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::format_command::command; +use crate::bindings::_helpers::format_command::{command, files}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - command::register(module) + command::register(module)?; + files::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs index 0ca023d5c2..3f3ce3f1e5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs @@ -164,7 +164,7 @@ fn native_graph_from_views(request: Option<&Bound<'_, PyDict>>) -> PyResult, downstream: &Bound<'_, PyAny>, ) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs index a4bfb862b6..d2bfcafadb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs @@ -4,21 +4,25 @@ use pyo3::exceptions::{PyKeyError, PyRecursionError, PyTypeError, PyValueError}; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::PyBytes; -use pyo3::{FromPyObject, PyRef, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::types::{PyBytes, PyDict}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_analysis::manifest::errors::{ManifestEncodeError, ManifestError}; use sqlbuild_analysis::manifest::main::build_manifest_json::build_manifest_json; use sqlbuild_analysis::manifest::models::ManifestInput; use sqlbuild_core::json::errors::JsonEmitError; +use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::bindings::_helpers::graph::views::native_graph_from_edges; use crate::bindings::_helpers::manifest::rows::{ ProjectRows, plan_input, project_rows, python_hooks, }; const FINGERPRINT_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.fingerprints.exceptions"; +const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; +const DATETIME_MODULE: &str = "datetime"; /// A built manifest: its `manifest.json` bytes, or the error `json.dumps` raised encoding it. #[pyclass(module = "sqlbuild._native", frozen)] @@ -56,36 +60,41 @@ fn python_error(error: ManifestError, py: Python<'_>) -> PyErr { } } -/// The manifest facts only Python knows: names, the generation stamp and hook identities. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct ManifestFacts<'py> { - project_name: String, - adapter_type: String, - /// `datetime.now(tz=UTC).isoformat()`. - generated_at: String, - /// `build_hook_identities(project.hook_functions)`. - hook_identities: Bound<'py, PyAny>, -} - -/// The manifest of `project` and the optional plan. +/// The manifest of `project` and the optional plan, over `graph` or `upstream_deps`/`downstream_deps`. #[pyfunction] -#[pyo3(signature = (*, project, plan_output, graph, facts))] -pub(crate) fn build_manifest( - project: &Bound<'_, PyAny>, - plan_output: &Bound<'_, PyAny>, - graph: PyRef<'_, NativeProjectGraph>, - facts: ManifestFacts<'_>, -) -> PyResult> { +#[pyo3(signature = (**options))] +pub(crate) fn build_manifest(options: Option<&Bound<'_, PyDict>>) -> PyResult> { + let request = Request::new("build_manifest", options)?; + let py: Python<'_> = request.py(); + let project: Bound<'_, PyAny> = request.get("project")?; + let plan_output: Bound<'_, PyAny> = request + .optional("plan_output")? + .unwrap_or_else(|| py.None().into_bound(py)); + let project_name: String = request.get("project_name")?.extract()?; + let adapter_type: String = request.get("adapter_type")?.extract()?; + let graph: Bound<'_, NativeProjectGraph> = match request.optional("graph")? { + Some(graph) if !graph.is_none() => graph.cast_into()?, + _ => Bound::new( + py, + native_graph_from_edges( + &request.get("upstream_deps")?, + &request.get("downstream_deps")?, + )?, + )?, + }; + let utc: Bound<'_, PyAny> = py.import(DATETIME_MODULE)?.getattr("UTC")?; + let generated_at: String = py + .import(DATETIME_MODULE)? + .getattr("datetime")? + .call_method1("now", (utc,))? + .call_method0("isoformat")? + .extract()?; + let hook_identities: Bound<'_, PyAny> = py + .import(HOOK_IDENTITIES_MODULE)? + .getattr("build_hook_identities")? + .call1((project.getattr("hook_functions")?,))?; compiler_guard(|| { - let py: Python<'_> = project.py(); - let ManifestFacts { - project_name, - adapter_type, - generated_at, - hook_identities, - } = facts; - let python_hooks = python_hooks(project, &hook_identities)?; + let python_hooks = python_hooks(&project, &hook_identities)?; let ProjectRows { run_id, sql_analysis_dialect, @@ -95,7 +104,7 @@ pub(crate) fn build_manifest( macros, macro_declarations, sql_hooks, - } = project_rows(project)?; + } = project_rows(&project)?; let input: ManifestInput = ManifestInput { project_name, adapter_type, @@ -109,8 +118,9 @@ pub(crate) fn build_manifest( macro_declarations, sql_hooks, python_hooks, - plan: plan_input(plan_output)?, + plan: plan_input(&plan_output)?, }; + let graph = graph.borrow(); let graph: &ProjectGraph = graph.graph(); let contents = py .detach(|| build_manifest_json(&input, graph)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs new file mode 100644 index 0000000000..59eac5be48 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs @@ -0,0 +1,128 @@ +//! The SQL expansion context compiler-integrated lint expands authored bodies with. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::build_sql_expansion_context; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.adapter.discovery.main.resolve_adapter"; +const ADAPTER_EXCEPTIONS_MODULE: &str = "sqlbuild.adapter.contract.exceptions"; +const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const DISCOVERY_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild.lint.exceptions"; +const COMPILE_FIRST_MESSAGE: &str = "compiler-integrated Rules check the SQL your project \ + actually produces, so the project must compile first"; + +/// Build the lint expansion context, reporting compile failures as `ProjectCompileError`. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_lint_expansion_context<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_lint_expansion_context", + options, + ( + &["project_dir"], + &[ + "value_renderer", + "discovered_inputs", + "declaration_scope", + "static_declaration_scope", + ], + ), + )?; + expansion_context(py, &arguments).or_else(|error| Err(compile_first_error(py, error)?)) +} + +fn expansion_context<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, +) -> PyResult> { + let project_dir: Bound<'py, PyAny> = arguments.value("project_dir")?; + let mut discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; + if discovered.is_none() { + let no: Bound<'py, PyAny> = PyBool::new(py, false).to_owned().into_any(); + discovered = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[ + ("project_dir", project_dir.clone()), + ("sql_analysis_enabled_override", no.clone()), + ("extract_output_column_locations", no), + ], + )?; + } + let mut renderer: Bound<'py, PyAny> = arguments.value("value_renderer")?; + if !renderer.is_truthy()? { + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + renderer = call_with( + &imported(py, RESOLVE_ADAPTER_MODULE, "resolve_adapter")?, + &[ + ("adapter_name", adapter_name), + ("project_dir", project_dir.clone()), + ], + )?; + } + build_sql_expansion_context( + py, + Some(keywords( + py, + &[ + ("project_dir", project_dir), + ("discovered_inputs", discovered), + ("value_renderer", renderer), + ("declaration_scope", arguments.value("declaration_scope")?), + ( + "static_declaration_scope", + arguments.value("static_declaration_scope")?, + ), + ], + )?), + ) +} + +/// Adapter, compile and discovery errors as `ProjectCompileError`; any other error unchanged. +fn compile_first_error(py: Python<'_>, error: PyErr) -> PyResult { + let value: Bound<'_, PyAny> = error.value(py).clone().into_any(); + let reported: bool = value.is_instance(&imported( + py, + ADAPTER_EXCEPTIONS_MODULE, + "AdapterUserError", + )?)? || value.is_instance(&imported( + py, + COMPILE_EXCEPTIONS_MODULE, + "CompileInputError", + )?)? || value.is_instance(&imported( + py, + DISCOVERY_EXCEPTIONS_MODULE, + "DiscoveryError", + )?)?; + if !reported { + return Ok(error); + } + let message: String = format!("{COMPILE_FIRST_MESSAGE}: {}", value.str()?); + let raised: PyErr = PyErr::from_value( + imported(py, LINT_EXCEPTIONS_MODULE, "ProjectCompileError")?.call1((message,))?, + ); + raised.set_cause(py, Some(error)); + Ok(raised) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_lint_expansion_context, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 3958390e03..0394c22076 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod dsl_headers; pub(crate) mod evaluation; pub(crate) mod findings; pub(crate) mod lint; +pub(crate) mod lint_expansion; pub(crate) mod lint_sql_bodies; pub(crate) mod lint_text; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index b57c779d18..021431b7eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -4,13 +4,15 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::rules::{ builtin_catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, evaluation, lint, - lint_sql_bodies, lint_text, results, rule_cache, rules_config, run_lint, skills, + lint_expansion, lint_sql_bodies, lint_text, results, rule_cache, rules_config, run_lint, + skills, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { evaluation::register(module)?; lint::register(module)?; lint_text::register(module)?; + lint_expansion::register(module)?; lint_sql_bodies::register(module)?; dsl_headers::register(module)?; custom_rule_evidence::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs index 928f1410c1..ec95af0ea3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs @@ -14,6 +14,8 @@ const INCREMENTAL_STRATEGY_CONFIG: &str = "incremental_strategy"; const CURSOR_CONFIG: &str = "cursor"; const UNIQUE_KEY_CONFIG: &str = "unique_key"; const UNIQUE_AUDIT_COLUMN_ARGUMENT: &str = "column"; +const RELATION_KIND_SEED: &str = "seed"; +const RELATION_KIND_SOURCE: &str = "source"; /// `(kind, name, materialized, mode, strategy, cursor, unique key, columns, audits)`. pub(crate) type RelationKeyRow = ( @@ -143,14 +145,66 @@ fn declared_specs<'py>(entry: &Bound<'py, PyAny>, name: &str) -> PyResult, - rows: Vec, -) -> BTreeMap>> { +#[pyo3(signature = (*, model_rows, seeds, sources))] +fn declared_relation_keys<'py>( + py: Python<'py>, + model_rows: &Bound<'py, PyAny>, + seeds: &Bound<'py, PyAny>, + sources: &Bound<'py, PyAny>, +) -> PyResult> { + let mut rows: Vec = Vec::new(); + for row in model_rows.try_iter()? { + rows.push(row?.extract()?); + } + for (kind, entries) in [(RELATION_KIND_SEED, seeds), (RELATION_KIND_SOURCE, sources)] { + for entry in entries.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + let name: String = entry.getattr("name")?.extract()?; + rows.push(entry_relation_key_row( + kind, + name, + &PyDict::new(py), + &entry, + )?); + } + } let relations: Vec = rows.into_iter().map(declared_relation).collect(); - py.detach(|| derive_keys(&relations)) + let keys: BTreeMap>> = py.detach(|| derive_keys(&relations)); + let found: Bound<'py, PyDict> = PyDict::new(py); + for (identity, unique_keys) in keys { + let mut tuples: Vec> = Vec::with_capacity(unique_keys.len()); + for key in unique_keys { + tuples.push(PyTuple::new(py, key)?); + } + found.set_item(identity, PyTuple::new(py, tuples)?)?; + } + Ok(found) +} + +/// Declared keys of a compiled project's models, seeds and sources. +#[pyfunction] +fn compiled_relation_keys<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + let mut model_rows: Vec> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + model_rows.push(model?.getattr("relation_key_row")?); + } + let mut seeds: Vec> = Vec::new(); + for seed in project.getattr("seeds")?.try_iter()? { + seeds.push(seed?.getattr("schema_entry")?); + } + let mut sources: Vec> = Vec::new(); + for source in project.getattr("sources")?.try_iter()? { + sources.push(source?.getattr("source_entry")?); + } + declared_relation_keys( + py, + PyTuple::new(py, model_rows)?.as_any(), + PyTuple::new(py, seeds)?.as_any(), + PyTuple::new(py, sources)?.as_any(), + ) } fn declared_relation(row: RelationKeyRow) -> DeclaredRelation { @@ -177,5 +231,6 @@ fn declared_relation(row: RelationKeyRow) -> DeclaredRelation { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(entry_relation_key_row, module)?)?; - module.add_function(wrap_pyfunction!(declared_relation_keys, module)?) + module.add_function(wrap_pyfunction!(declared_relation_keys, module)?)?; + module.add_function(wrap_pyfunction!(compiled_relation_keys, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs index 0dfbbdf000..f2a30c19e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs @@ -14,8 +14,9 @@ use sqlbuild_core::format_command::main::collect_lint_files::collect_lint_files; use sqlbuild_rules::lint_text::main::first_header_only::first_header_only; use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, + KeywordArguments, call_with, imported, keywords, }; +use crate::bindings::_helpers::rules::lint_expansion::build_lint_expansion_context; const NATIVE_MODULE: &str = "sqlbuild._native"; const EXPANSION_MODULE: &str = "sqlbuild.lint._helpers.expansion"; @@ -167,13 +168,16 @@ fn expansion_context<'py>( if !run.config.getattr("native_enabled")?.is_truthy()? { return Ok(py.None().into_bound(py)); } - call_with( - &imported(py, EXPANSION_MODULE, "build_lint_expansion_context")?, - &[ - ("project_dir", run.project_dir.clone()), - ("value_renderer", arguments.value("value_renderer")?), - ("discovered_inputs", arguments.value("discovered_inputs")?), - ], + build_lint_expansion_context( + py, + Some(keywords( + py, + &[ + ("project_dir", run.project_dir.clone()), + ("value_renderer", arguments.value("value_renderer")?), + ("discovered_inputs", arguments.value("discovered_inputs")?), + ], + )?), ) } diff --git a/fensu.toml b/fensu.toml index 7249edb97e..df00197378 100644 --- a/fensu.toml +++ b/fensu.toml @@ -93,6 +93,21 @@ rule = "FFR309" path = "src/sqlbuild/compiler/compile/__init__.py" reason = "Compilation runs natively; the package keeps the compile models, views, constants, errors, the diagnostic collector and the user-macro and Python-resource glue the native compiler calls back into." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/references/__init__.py" +reason = "SQL references are scanned natively; the package keeps the reference models, kinds and the reference-call patterns the Python planner still matches." + +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/resource_names/__init__.py" +reason = "Resource identities are validated natively; the package keeps the error class the native check raises." + +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/sql_values/__init__.py" +reason = "Typed SQL values are normalized and state literals rendered natively; the package keeps the value models, types, limits and errors the native bindings build and raise." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/refactoring/__init__.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 78bb2a5ee0..74b78d32eb 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -224,7 +224,6 @@ from sqlbuild.compiler.planner.types import ( from sqlbuild.compiler.project_assembly.models import NativeProjectResources from sqlbuild.compiler.python_nodes.models import ( PythonNodeGraph, - PythonNodeIdentity, PythonSqlRunLifecyclePlan, PythonSqlRunSelection, ) @@ -318,7 +317,7 @@ from sqlbuild.integrations.dbt.models import ( ) from sqlbuild.integrations.dbt.types import DbtInteropCommand from sqlbuild.lint.models import HeaderSpan, LintBody, LintConfig, LintRunResult, LintViolation -from sqlbuild.lint.types import RelationKeyRow +from sqlbuild.lint.types import RelationKeyRow, RelationKeys from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.models import DisplayOptions from sqlbuild.presentation.types import CompletionState @@ -355,7 +354,7 @@ from sqlbuild.spec.contracts.types import ( SourceFreshnessValueKind, WarehouseGroup, ) -from sqlbuild.sql_values.models import SqlValue +from sqlbuild.sql_values.models import SqlValue, SqlValueLimits from sqlbuild.sql_values.types import CollectionRendering, StateSqlValueType BUILD_IDENTITY: str @@ -1146,26 +1145,38 @@ class NativeModelLoop: def relation_key_row( kind: str, name: str, values: dict[str, object], entry: object ) -> RelationKeyRow: ... -def declared_relation_keys(rows: list[RelationKeyRow]) -> dict[str, list[list[str]]]: ... +def declared_relation_keys( + *, + model_rows: Iterable[RelationKeyRow], + seeds: Iterable[SchemaSeedEntry], + sources: Iterable[SourceEntry], +) -> RelationKeys: ... +def compiled_relation_keys(project: CompiledProjectView) -> RelationKeys: ... +def build_lint_expansion_context( + *, + project_dir: Path, + value_renderer: TypedSqlValueRenderer | None = None, + discovered_inputs: DiscoveredProjectInputs | None = None, + declaration_scope: DeclarationScopeBuild | None = None, + static_declaration_scope: DeclarationScopeBuild | None = None, +) -> SqlExpansionContext: ... +def write_atomically(*, path: Path, contents: str) -> None: ... def build_dag_json( *, graph: ProjectGraph, project_name: str, python_graph: PythonNodeGraph | None = None ) -> str: ... -class _ManifestFacts(TypedDict): - project_name: str - adapter_type: str - generated_at: str - hook_identities: Mapping[str, PythonNodeIdentity] - class NativeManifest: def contents(self) -> bytes: ... def build_manifest( *, project: CompiledProjectView, - plan_output: PlanOutput | None, - graph: NativeProjectGraph, - facts: _ManifestFacts, + plan_output: PlanOutput | None = None, + project_name: str, + adapter_type: str, + graph: NativeProjectGraph | None = None, + upstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]] | None = None, + downstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]] | None = None, ) -> NativeManifest: ... class NativeProjectGraph: @@ -2624,9 +2635,14 @@ def matching_dynamic_families( *, families: tuple[SchemaDynamicColumnFamily, ...], column_name: str ) -> tuple[SchemaDynamicColumnFamily, ...]: ... def assert_no_unresolved_sql_markers(*, sql: str, context: str) -> None: ... -def normalize_typed_sql_value( - raw_value: object, typing: tuple[str, str | None], limits: tuple[int, int, int], / -) -> object: ... +def normalize_sql_value( + *, + raw_value: object, + context: str, + explicit_type: str | None = None, + limits: SqlValueLimits | None = None, +) -> SqlValue: ... +def render_state_sql_literal(*, value: object | None, declared_type: StateSqlValueType) -> str: ... def model_header_declarations( header: tuple[object | None, object | None], model_name: str, relative_path: Path, / ) -> tuple[tuple[object, ...], tuple[object, ...]]: ... @@ -2709,9 +2725,10 @@ def find_python_string_locations( reason: str, context: str | None = None, ) -> tuple[ManualLocation, ...]: ... -def resource_identity_suggestion( - name: str, private_identity: bool, / -) -> tuple[str, str] | None: ... +def validate_resource_identity( + *, name: str, kind: str, path: Path, private_identity: bool = False +) -> None: ... +def function_node_type(*, return_columns: tuple[object, ...]) -> str: ... def load_project_config_file(project_dir: Path, /) -> ProjectConfig: ... def load_local_config_file(project_dir: Path, /) -> LocalConfig: ... def resolve_config_file_path(project_dir: Path, local: bool, /) -> Path: ... diff --git a/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py b/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py index c6bb91f05b..7edd187e8a 100644 --- a/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py +++ b/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py @@ -4,6 +4,7 @@ from collections.abc import Callable +from sqlbuild._native import render_state_sql_literal from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.compiler.source_freshness.constants import ( COLUMN_DATA_VERSION, @@ -21,7 +22,6 @@ SOURCE_FRESHNESS_TABLE_NAME, ) from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal def render_insert_source_freshness_records_sql( diff --git a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py index e09fa8cee4..7473eecc9d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py @@ -16,6 +16,7 @@ from typing import cast import sqlbuild._native as _native +from sqlbuild._native import validate_resource_identity from sqlbuild.compiler.compile.exceptions import ( CompileInputError, MacroDeclarationLookupError, @@ -33,9 +34,6 @@ DiscoveredMacroFile, EnumDeclaration, ) -from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( - validate_resource_identity, -) from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, diff --git a/src/sqlbuild/compiler/contract_adoption/main/_write.py b/src/sqlbuild/compiler/contract_adoption/main/_write.py index fd0219ff50..7050ced037 100644 --- a/src/sqlbuild/compiler/contract_adoption/main/_write.py +++ b/src/sqlbuild/compiler/contract_adoption/main/_write.py @@ -11,6 +11,7 @@ edit_contract_model, edit_contract_source, remaining_contract_findings, + write_atomically, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( @@ -28,7 +29,6 @@ ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.lint.main.write_atomically import write_atomically def write_contracts( diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index 588848f6b6..c981bd9dc3 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Any, get_type_hints import sqlbuild._native as _native +from sqlbuild._native import validate_resource_identity from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import ( project_relative_path, @@ -42,9 +43,6 @@ DiscoveredPythonNodeFunctions, DiscoveredTaskFunction, ) -from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( - validate_resource_identity, -) from sqlbuild.provider.exceptions import ProviderInputError from sqlbuild.python_nodes.main.read_asset_definition import read_asset_definition from sqlbuild.python_nodes.main.read_audit_factory_definition import ( diff --git a/src/sqlbuild/compiler/manifest/__init__.py b/src/sqlbuild/compiler/manifest/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/manifest/main/__init__.py b/src/sqlbuild/compiler/manifest/main/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py b/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py deleted file mode 100644 index 2f032de89e..0000000000 --- a/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Build a dbt-compatible manifest.json from a compiled project and its native graph.""" - -from __future__ import annotations - -from datetime import UTC, datetime - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.python_nodes.main._hook_identities import build_hook_identities - - -def build_graph_manifest( - *, - project: CompiledProjectView, - plan_output: PlanOutput | None, - project_name: str, - adapter_type: str, - graph: _native.NativeProjectGraph, -) -> _native.NativeManifest: - """Build the manifest with parent and child maps read from the native lineage graph.""" - - return _native.build_manifest( - project=project, - plan_output=plan_output, - graph=graph, - facts={ - "project_name": project_name, - "adapter_type": adapter_type, - "generated_at": datetime.now(tz=UTC).isoformat(), - "hook_identities": build_hook_identities(project.hook_functions), - }, - ) diff --git a/src/sqlbuild/compiler/manifest/main/build.py b/src/sqlbuild/compiler/manifest/main/build.py deleted file mode 100644 index 63ab9d03c3..0000000000 --- a/src/sqlbuild/compiler/manifest/main/build.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Build a dbt-compatible manifest.json from compiled project state.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild._native import native_graph_from_edges -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.manifest.main._build_graph_manifest import build_graph_manifest -from sqlbuild.compiler.planner.models import PlanOutput - - -def build_manifest( - *, - project: CompiledProjectView, - plan_output: PlanOutput | None = None, - project_name: str, - adapter_type: str, - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> _native.NativeManifest: - """Build a full dbt v12-compatible manifest from caller-built edge maps.""" - - graph: _native.NativeProjectGraph = native_graph_from_edges( - upstream=upstream_deps, downstream=downstream_deps - ) - return build_graph_manifest( - project=project, - plan_output=plan_output, - project_name=project_name, - adapter_type=adapter_type, - graph=graph, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py index 1f7a3c182c..ea4cf8a5be 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py @@ -27,26 +27,25 @@ SQL_IDENTIFIER_LEADING_CHARACTERS, ) from sqlbuild.compiler.planner.models import CursorBounds -from sqlbuild.compiler.references.main._quoted_reference_call_pattern import ( - quoted_reference_call_pattern, +from sqlbuild.compiler.references.constants import ( + DBT_REF_CALL_PREFIX, + REF_PATTERN, + SEED_PATTERN, + TABLE_FUNCTION_PATTERN, + UDF_PATTERN, ) -from sqlbuild.compiler.references.main._reference_call_prefix_pattern_text import ( - reference_call_prefix_pattern_text, -) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind +from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -_REF_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.REF) -_SEED_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.SEED) +_REF_PATTERN: re.Pattern[str] = REF_PATTERN +_SEED_PATTERN: re.Pattern[str] = SEED_PATTERN _DBT_REF_PATTERN: re.Pattern[str] = re.compile( - rf'{reference_call_prefix_pattern_text(SqlReferenceKind.DBT_REF)}\s*"([^"]+)"\s*' + rf'{DBT_REF_CALL_PREFIX}\s*"([^"]+)"\s*' r'(?:,\s*"([^"]+)"\s*)?\)', re.IGNORECASE, ) -_UDF_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.UDF) -_TABLE_FUNCTION_PATTERN: re.Pattern[str] = quoted_reference_call_pattern( - SqlReferenceKind.TABLE_FUNCTION -) +_UDF_PATTERN: re.Pattern[str] = UDF_PATTERN +_TABLE_FUNCTION_PATTERN: re.Pattern[str] = TABLE_FUNCTION_PATTERN _REFERENCE_SCAN_CONTEXT: str = "SQL reference resolution" _CLAUSE_KEYWORDS: frozenset[str] = frozenset( { diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py b/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py index 1ab40f1157..729848a6f7 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py @@ -18,13 +18,10 @@ from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import CursorBounds from sqlbuild.compiler.planner.types import ContractPolicy -from sqlbuild.compiler.references.main._quoted_reference_call_pattern import ( - quoted_reference_call_pattern, -) -from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.references.constants import SOURCE_PATTERN from sqlbuild.spec.contracts.models import SourceColumnEntry, SourceEntry -_SOURCE_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.SOURCE) +_SOURCE_PATTERN: re.Pattern[str] = SOURCE_PATTERN _DERIVED_TABLE_ALIAS_KEYWORDS: frozenset[str] = frozenset( { "APPLY", diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py index b4aed7b733..385e1dcffb 100644 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py +++ b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py @@ -79,26 +79,21 @@ FixtureKey, ScenarioArtifactKind, ) -from sqlbuild.compiler.references.main._reference_call_prefix_pattern_text import ( - reference_call_prefix_pattern_text, +from sqlbuild.compiler.references.constants import ( + DBT_REF_CALL_PREFIX, + REF_CALL_PREFIX, + SEED_CALL_PREFIX, + SOURCE_CALL_PREFIX, ) -from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import SourceEntry _NAME_ARGUMENT_PATTERN_TEXT: str = r'"(?P[^"]+)"\)' -_REF_PATTERN: re.Pattern[str] = re.compile( - reference_call_prefix_pattern_text(SqlReferenceKind.REF) + _NAME_ARGUMENT_PATTERN_TEXT -) -_SEED_PATTERN: re.Pattern[str] = re.compile( - reference_call_prefix_pattern_text(SqlReferenceKind.SEED) + _NAME_ARGUMENT_PATTERN_TEXT -) -_SOURCE_PATTERN: re.Pattern[str] = re.compile( - reference_call_prefix_pattern_text(SqlReferenceKind.SOURCE) + _NAME_ARGUMENT_PATTERN_TEXT -) +_REF_PATTERN: re.Pattern[str] = re.compile(REF_CALL_PREFIX + _NAME_ARGUMENT_PATTERN_TEXT) +_SEED_PATTERN: re.Pattern[str] = re.compile(SEED_CALL_PREFIX + _NAME_ARGUMENT_PATTERN_TEXT) +_SOURCE_PATTERN: re.Pattern[str] = re.compile(SOURCE_CALL_PREFIX + _NAME_ARGUMENT_PATTERN_TEXT) _DBT_REF_PATTERN: re.Pattern[str] = re.compile( - reference_call_prefix_pattern_text(SqlReferenceKind.DBT_REF) - + r'"(?P[^"]+)"(?:\s*,\s*"(?P[^"]+)")?\)' + DBT_REF_CALL_PREFIX + r'"(?P[^"]+)"(?:\s*,\s*"(?P[^"]+)")?\)' ) diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py index d7f40f15ea..78a348b44a 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py @@ -23,16 +23,14 @@ SOURCE_DEFERRAL_CONTEXT_FIELDS, ) from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.references.main._quoted_reference_call_pattern import ( - quoted_reference_call_pattern, -) +from sqlbuild.compiler.references.constants import SOURCE_PATTERN from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.python_nodes.types import SqlResourceRefKind from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceEntry, TargetConfig _CTX_PATTERN: re.Pattern[str] = re.compile(r"\$\{CTX:([^}]+)\}") _VAR_PATTERN: re.Pattern[str] = re.compile(r"\$\{([^}:]+)\}") -_SOURCE_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.SOURCE) +_SOURCE_PATTERN: re.Pattern[str] = SOURCE_PATTERN def build_source_read_map( diff --git a/src/sqlbuild/compiler/references/_helpers/__init__.py b/src/sqlbuild/compiler/references/_helpers/__init__.py deleted file mode 100644 index 2d2301793d..0000000000 --- a/src/sqlbuild/compiler/references/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Internal SQL reference helpers.""" diff --git a/src/sqlbuild/compiler/references/_helpers/patterns.py b/src/sqlbuild/compiler/references/_helpers/patterns.py deleted file mode 100644 index c50df5aabb..0000000000 --- a/src/sqlbuild/compiler/references/_helpers/patterns.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Regex helpers for SQLBuild reference function syntax.""" - -from __future__ import annotations - -import re - -from sqlbuild.compiler.references.types import SqlReferenceKind - - -def reference_call_prefix_pattern_text(kind: SqlReferenceKind) -> str: - return rf"{re.escape(kind.function_name)}\(" - - -def quoted_reference_call_pattern_text(kind: SqlReferenceKind) -> str: - return rf'{reference_call_prefix_pattern_text(kind)}"([^"]+)"\)' - - -def quoted_reference_call_pattern(kind: SqlReferenceKind) -> re.Pattern[str]: - return re.compile(quoted_reference_call_pattern_text(kind)) diff --git a/src/sqlbuild/compiler/references/constants.py b/src/sqlbuild/compiler/references/constants.py index 494af6394e..68c894a12a 100644 --- a/src/sqlbuild/compiler/references/constants.py +++ b/src/sqlbuild/compiler/references/constants.py @@ -4,9 +4,19 @@ import re -from sqlbuild.compiler.references._helpers.patterns import quoted_reference_call_pattern from sqlbuild.compiler.references.types import SqlReferenceKind -REF_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.REF) -SEED_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.SEED) -SOURCE_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.SOURCE) +REF_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.REF.function_name)}\(" +SEED_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.SEED.function_name)}\(" +SOURCE_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.SOURCE.function_name)}\(" +DBT_REF_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.DBT_REF.function_name)}\(" +UDF_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.UDF.function_name)}\(" +TABLE_FUNCTION_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.TABLE_FUNCTION.function_name)}\(" +_QUOTED_NAME_ARGUMENT: str = r'"([^"]+)"\)' +REF_PATTERN: re.Pattern[str] = re.compile(REF_CALL_PREFIX + _QUOTED_NAME_ARGUMENT) +SEED_PATTERN: re.Pattern[str] = re.compile(SEED_CALL_PREFIX + _QUOTED_NAME_ARGUMENT) +SOURCE_PATTERN: re.Pattern[str] = re.compile(SOURCE_CALL_PREFIX + _QUOTED_NAME_ARGUMENT) +UDF_PATTERN: re.Pattern[str] = re.compile(UDF_CALL_PREFIX + _QUOTED_NAME_ARGUMENT) +TABLE_FUNCTION_PATTERN: re.Pattern[str] = re.compile( + TABLE_FUNCTION_CALL_PREFIX + _QUOTED_NAME_ARGUMENT +) diff --git a/src/sqlbuild/compiler/references/main/__init__.py b/src/sqlbuild/compiler/references/main/__init__.py deleted file mode 100644 index 0045272af3..0000000000 --- a/src/sqlbuild/compiler/references/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Public SQL reference entries.""" diff --git a/src/sqlbuild/compiler/references/main/_quoted_reference_call_pattern.py b/src/sqlbuild/compiler/references/main/_quoted_reference_call_pattern.py deleted file mode 100644 index 3c8ef9af89..0000000000 --- a/src/sqlbuild/compiler/references/main/_quoted_reference_call_pattern.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public quoted SQL reference pattern entry.""" - -from __future__ import annotations - -import re - -from sqlbuild.compiler.references._helpers.patterns import ( - quoted_reference_call_pattern as _quoted_reference_call_pattern, -) -from sqlbuild.compiler.references.types import SqlReferenceKind - - -def quoted_reference_call_pattern(kind: SqlReferenceKind) -> re.Pattern[str]: - return _quoted_reference_call_pattern(kind) diff --git a/src/sqlbuild/compiler/references/main/_reference_call_prefix_pattern_text.py b/src/sqlbuild/compiler/references/main/_reference_call_prefix_pattern_text.py deleted file mode 100644 index 1f9b151d12..0000000000 --- a/src/sqlbuild/compiler/references/main/_reference_call_prefix_pattern_text.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public SQL reference prefix pattern entry.""" - -from __future__ import annotations - -from sqlbuild.compiler.references._helpers.patterns import ( - reference_call_prefix_pattern_text as _reference_call_prefix_pattern_text, -) -from sqlbuild.compiler.references.types import SqlReferenceKind - - -def reference_call_prefix_pattern_text(kind: SqlReferenceKind) -> str: - return _reference_call_prefix_pattern_text(kind) diff --git a/src/sqlbuild/compiler/resource_names/main/__init__.py b/src/sqlbuild/compiler/resource_names/main/__init__.py deleted file mode 100644 index 2afe5c8342..0000000000 --- a/src/sqlbuild/compiler/resource_names/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Resource name validation entrypoints.""" diff --git a/src/sqlbuild/compiler/resource_names/main/_validate_resource_identity.py b/src/sqlbuild/compiler/resource_names/main/_validate_resource_identity.py deleted file mode 100644 index 9181d26aa2..0000000000 --- a/src/sqlbuild/compiler/resource_names/main/_validate_resource_identity.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Canonical resource identity validation entrypoint.""" - -from __future__ import annotations - -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError - - -def validate_resource_identity( - *, name: str, kind: str, path: Path, private_identity: bool = False -) -> None: - """Reject one identity unless it uses canonical lowercase snake_case.""" - - failure: tuple[str, str] | None = _native.resource_identity_suggestion(name, private_identity) - if failure is None: - return - corrected_name, help_text = failure - raise ResourceIdentityError( - f"Invalid {kind} identity '{name}' in {path}; use snake_case '{corrected_name}'", - help=help_text, - ) diff --git a/src/sqlbuild/compiler/resource_names/main/function_node_type.py b/src/sqlbuild/compiler/resource_names/main/function_node_type.py deleted file mode 100644 index 6caf6e2eca..0000000000 --- a/src/sqlbuild/compiler/resource_names/main/function_node_type.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Resolve compiled resource type for SQL functions.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledResourceType - - -def _function_resource_type(*, return_columns: tuple[object, ...]) -> CompiledResourceType: - """Return the compiled resource type for a SQL function shape.""" - - return CompiledResourceType.TABLE_FN if return_columns else CompiledResourceType.UDF - - -def function_node_type(*, return_columns: tuple[object, ...]) -> str: - """Return the public node type for a SQL function shape.""" - - return _function_resource_type(return_columns=return_columns).value diff --git a/src/sqlbuild/executor/audit_results/_helpers/sql.py b/src/sqlbuild/executor/audit_results/_helpers/sql.py index 296d7a1a36..986206bad2 100644 --- a/src/sqlbuild/executor/audit_results/_helpers/sql.py +++ b/src/sqlbuild/executor/audit_results/_helpers/sql.py @@ -4,6 +4,7 @@ from collections.abc import Callable, Sequence +from sqlbuild._native import render_state_sql_literal from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( render_state_table_create_sql, @@ -54,7 +55,6 @@ COLUMN_VIOLATION_COUNT, ) from sqlbuild.executor.audit_results.models import AuditResultRecord -from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal _REQUIRED_COLUMNS: frozenset[str] = frozenset( { diff --git a/src/sqlbuild/executor/functions/main/_execute.py b/src/sqlbuild/executor/functions/main/_execute.py index 1daeda2723..b90c1b264e 100644 --- a/src/sqlbuild/executor/functions/main/_execute.py +++ b/src/sqlbuild/executor/functions/main/_execute.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( compute_function_definition_hash, + function_node_type, write_fingerprint, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -16,7 +17,6 @@ from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import FunctionPlanEntry -from sqlbuild.compiler.resource_names.main.function_node_type import function_node_type from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.errors.contracts.main.error_code import error_code from sqlbuild.errors.contracts.main.error_help import error_help diff --git a/src/sqlbuild/executor/janitor_events/_helpers/sql.py b/src/sqlbuild/executor/janitor_events/_helpers/sql.py index a3f3a3202e..5d85cae77f 100644 --- a/src/sqlbuild/executor/janitor_events/_helpers/sql.py +++ b/src/sqlbuild/executor/janitor_events/_helpers/sql.py @@ -4,6 +4,7 @@ from collections.abc import Callable +from sqlbuild._native import render_state_sql_literal from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( render_state_table_create_sql, @@ -29,7 +30,6 @@ JANITOR_EVENTS_TABLE_NAME, ) from sqlbuild.executor.janitor_events.models import JanitorEventRecord -from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal def build_qualified_table_name( diff --git a/src/sqlbuild/executor/node_results/_helpers/sql.py b/src/sqlbuild/executor/node_results/_helpers/sql.py index be338e043f..d8669e4ee9 100644 --- a/src/sqlbuild/executor/node_results/_helpers/sql.py +++ b/src/sqlbuild/executor/node_results/_helpers/sql.py @@ -4,6 +4,7 @@ from collections.abc import Callable +from sqlbuild._native import render_state_sql_literal from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( render_state_table_create_sql, @@ -27,7 +28,6 @@ NODE_RESULTS_TABLE_NAME, ) from sqlbuild.executor.node_results.models import NodeResultQuery, NodeResultRecord -from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal _REQUIRED_NODE_RESULT_COLUMNS: frozenset[str] = frozenset( { diff --git a/src/sqlbuild/lint/_helpers/expansion.py b/src/sqlbuild/lint/_helpers/expansion.py index ea60fed18b..b06a8229e5 100644 --- a/src/sqlbuild/lint/_helpers/expansion.py +++ b/src/sqlbuild/lint/_helpers/expansion.py @@ -8,25 +8,16 @@ import sqlbuild._native as _native from sqlbuild._native import ( - build_sql_expansion_context, - discover_project_inputs, expand_sql_with_spans, external_identifiers_for_headers, - resolve_effective_adapter_name, ) -from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter from sqlbuild.compiler.compile.constants import MACRO_TOKEN from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompiledSqlExpansion, - DeclarationScopeBuild, ExpansionSpan, SqlExpansionContext, ) -from sqlbuild.compiler.compile.types import TypedSqlValueRenderer -from sqlbuild.compiler.discovery.exceptions import DiscoveryError -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.lint._helpers.headers import lint_body_ranges, lint_file_role from sqlbuild.lint.classes.stop_context import LintStopContext @@ -56,57 +47,6 @@ ) -def build_lint_expansion_context( - *, - project_dir: Path, - value_renderer: TypedSqlValueRenderer | None = None, - discovered_inputs: DiscoveredProjectInputs | None = None, - declaration_scope: DeclarationScopeBuild | None = None, - static_declaration_scope: DeclarationScopeBuild | None = None, -) -> SqlExpansionContext: - """Build the expansion context, reporting compile failures as lint failures.""" - - try: - effective_discovered_inputs: DiscoveredProjectInputs = ( - discovered_inputs - if discovered_inputs is not None - else discover_project_inputs( - project_dir=project_dir, - sql_analysis_enabled_override=False, - extract_output_column_locations=False, - ) - ) - effective_renderer: TypedSqlValueRenderer = value_renderer or _resolve_value_renderer( - project_dir=project_dir, - discovered_inputs=effective_discovered_inputs, - ) - return build_sql_expansion_context( - project_dir=project_dir, - discovered_inputs=effective_discovered_inputs, - value_renderer=effective_renderer, - declaration_scope=declaration_scope, - static_declaration_scope=static_declaration_scope, - ) - except (AdapterUserError, CompileInputError, DiscoveryError) as error: - raise ProjectCompileError( - "compiler-integrated Rules check the SQL your project actually produces, " - "so the project must " - f"compile first: {error}" - ) from error - - -def _resolve_value_renderer( - *, project_dir: Path, discovered_inputs: DiscoveredProjectInputs -) -> TypedSqlValueRenderer: - return resolve_adapter( - adapter_name=resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ), - project_dir=project_dir, - ) - - def prepare_lint_body( *, role: LintFileRole, diff --git a/src/sqlbuild/lint/_helpers/fixes.py b/src/sqlbuild/lint/_helpers/fixes.py index 39c8893de8..0dbc1b6233 100644 --- a/src/sqlbuild/lint/_helpers/fixes.py +++ b/src/sqlbuild/lint/_helpers/fixes.py @@ -2,13 +2,11 @@ from __future__ import annotations -import os -import tempfile from collections import Counter from dataclasses import dataclass, field, replace from pathlib import Path -from sqlbuild._native import build_compiled_project, run_lint +from sqlbuild._native import build_compiled_project, compiled_relation_keys, run_lint from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView, TypedSqlValueRenderer @@ -21,7 +19,6 @@ ) from sqlbuild.lint.exceptions import ProjectCompileError from sqlbuild.lint.main.compile_facts import compile_facts -from sqlbuild.lint.main.compiled_relation_keys import compiled_relation_keys from sqlbuild.lint.main.fix_verdict import fix_verdict from sqlbuild.lint.models import ( CompileFacts, @@ -292,20 +289,6 @@ def _decline( ) -def write_atomically(*, path: Path, contents: str) -> None: - descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) - temporary_path: Path = Path(temporary_name) - try: - os.fchmod(descriptor, path.stat().st_mode) - with os.fdopen(descriptor, "w", encoding="utf-8", newline="") as handle: - _ = handle.write(contents) - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary_path, path) - finally: - temporary_path.unlink(missing_ok=True) - - def plan_rule_fixes( *, project_dir: Path, diff --git a/src/sqlbuild/lint/main/build_expansion_context.py b/src/sqlbuild/lint/main/build_expansion_context.py deleted file mode 100644 index 3a3cefe84f..0000000000 --- a/src/sqlbuild/lint/main/build_expansion_context.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Prepare invocation-local SQL expansion evidence for compiler-integrated lint.""" - -from pathlib import Path - -from sqlbuild.compiler.compile.models import DeclarationScopeBuild, SqlExpansionContext -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.lint._helpers.expansion import build_lint_expansion_context - - -def build_expansion_context( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - declaration_scope: DeclarationScopeBuild | None = None, - static_declaration_scope: DeclarationScopeBuild | None = None, -) -> SqlExpansionContext: - return build_lint_expansion_context( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - declaration_scope=declaration_scope, - static_declaration_scope=static_declaration_scope, - ) diff --git a/src/sqlbuild/lint/main/compiled_relation_keys.py b/src/sqlbuild/lint/main/compiled_relation_keys.py deleted file mode 100644 index de49299728..0000000000 --- a/src/sqlbuild/lint/main/compiled_relation_keys.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Declared unique keys of a compiled project's relations.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.lint.main.declared_relation_keys import declared_relation_keys -from sqlbuild.lint.types import RelationKeys - - -def compiled_relation_keys(project: CompiledProjectView) -> RelationKeys: - """Collect declared keys of compiled models, seeds and sources.""" - - return declared_relation_keys( - model_rows=(model.relation_key_row for model in project.models), - seeds=(seed.schema_entry for seed in project.seeds), - sources=(source.source_entry for source in project.sources), - ) diff --git a/src/sqlbuild/lint/main/declared_relation_keys.py b/src/sqlbuild/lint/main/declared_relation_keys.py deleted file mode 100644 index 37237157d4..0000000000 --- a/src/sqlbuild/lint/main/declared_relation_keys.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Declared unique keys of project relations, for ranking determinism proofs.""" - -from __future__ import annotations - -from collections.abc import Iterable - -import sqlbuild._native as _native -from sqlbuild.lint.constants import RELATION_KIND_SEED, RELATION_KIND_SOURCE -from sqlbuild.lint.types import RelationKeyRow, RelationKeys -from sqlbuild.spec.contracts.models import SchemaSeedEntry, SourceEntry - - -def declared_relation_keys( - *, - model_rows: Iterable[RelationKeyRow], - seeds: Iterable[SchemaSeedEntry], - sources: Iterable[SourceEntry], -) -> RelationKeys: - """Keys that prove one row per key, from model key rows and seed and source entries.""" - - rows: list[RelationKeyRow] = [ - *model_rows, - *(_native.relation_key_row(RELATION_KIND_SEED, seed.name, {}, seed) for seed in seeds), - *( - _native.relation_key_row(RELATION_KIND_SOURCE, source.name, {}, source) - for source in sources - ), - ] - keys: RelationKeys = {} - for identity, found in _native.declared_relation_keys(rows).items(): - keys[identity] = tuple(map(tuple, found)) - return keys diff --git a/src/sqlbuild/lint/main/write_atomically.py b/src/sqlbuild/lint/main/write_atomically.py deleted file mode 100644 index 96599a0f29..0000000000 --- a/src/sqlbuild/lint/main/write_atomically.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public atomic source-file writer.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.lint._helpers.fixes import write_atomically as _write_atomically - - -def write_atomically(*, path: Path, contents: str) -> None: - """Replace a text file atomically.""" - - _write_atomically(path=path, contents=contents) diff --git a/src/sqlbuild/microbatches/_helpers/sql.py b/src/sqlbuild/microbatches/_helpers/sql.py index cf549ba41a..7f8928e42c 100644 --- a/src/sqlbuild/microbatches/_helpers/sql.py +++ b/src/sqlbuild/microbatches/_helpers/sql.py @@ -4,6 +4,7 @@ from collections.abc import Callable +from sqlbuild._native import render_state_sql_literal from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec from sqlbuild.microbatches.constants import ( @@ -14,7 +15,6 @@ ) from sqlbuild.microbatches.exceptions import MicrobatchStateError from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope -from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal from sqlbuild.sql_values.types import StateSqlValueType diff --git a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py index 9e65027fea..962a16c4ad 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py @@ -7,7 +7,7 @@ from pathlib import Path from threading import Event -from sqlbuild._native import run_lint +from sqlbuild._native import build_lint_expansion_context, run_lint from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledSqlExpansion, @@ -18,7 +18,6 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.lint.classes.stop_context import LintStopContext -from sqlbuild.lint.main.build_expansion_context import build_expansion_context from sqlbuild.lint.models import LintConfig, LintRunResult from sqlbuild.rule_engine._helpers.engine.custom_dispatch import evaluate_native, start_custom_rules from sqlbuild.rule_engine._helpers.run.rules import ( @@ -135,7 +134,7 @@ def _prepare_sql_lint( ) -> PreparedSqlLintResult: stop_token: Token[Event | None] = LintStopContext.active.set(stop) try: - context: SqlExpansionContext = build_expansion_context( + context: SqlExpansionContext = build_lint_expansion_context( project_dir=project_dir, discovered_inputs=discovered_inputs, static_declaration_scope=static_declaration_scope, diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index e4bed6d647..c5cf83ab5c 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -8,7 +8,10 @@ from pathlib import Path from sqlbuild._native import ( + build_lint_expansion_context, collect_lint_project_files, + compiled_relation_keys, + declared_relation_keys, encode_sql_rule_findings, load_sql_rule_cache, lookup_sql_rule_cache, @@ -44,9 +47,6 @@ SAFE_RULE_FIX_CODES, ) from sqlbuild.lint.exceptions import NativeLintError -from sqlbuild.lint.main.build_expansion_context import build_expansion_context -from sqlbuild.lint.main.compiled_relation_keys import compiled_relation_keys -from sqlbuild.lint.main.declared_relation_keys import declared_relation_keys from sqlbuild.lint.models import LintConfig, LintRunResult, LintViolation from sqlbuild.lint.types import RelationKeys from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules @@ -577,7 +577,7 @@ def _lint_expansion_context( if expansion_reuse is not None and expansion_reuse.discovered_inputs is discovered_inputs else None ) - return build_expansion_context( + return build_lint_expansion_context( project_dir=project_dir, discovered_inputs=discovered_inputs, declaration_scope=declaration_scope, diff --git a/src/sqlbuild/sql_values/_helpers/__init__.py b/src/sqlbuild/sql_values/_helpers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/sql_values/_helpers/state_literal.py b/src/sqlbuild/sql_values/_helpers/state_literal.py deleted file mode 100644 index 1f78d80dfc..0000000000 --- a/src/sqlbuild/sql_values/_helpers/state_literal.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Implementation helpers for typed warehouse-state SQL literals.""" - -from __future__ import annotations - -import json -from datetime import UTC, date, datetime - -from sqlbuild.sql_values.exceptions import SqlValueRenderingError -from sqlbuild.sql_values.types import StateSqlValueType - - -def render_state_sql_literal(*, value: object | None, declared_type: StateSqlValueType) -> str: - """Render a Python value according to its declared state-column type.""" - - if value is None: - return _typed_null(declared_type=declared_type) - match declared_type: - case StateSqlValueType.STRING: - if not isinstance(value, str): - raise SqlValueRenderingError("state string columns require str values") - return _quoted(value=value) - case StateSqlValueType.INTEGER: - if isinstance(value, bool) or not isinstance(value, int): - raise SqlValueRenderingError("state integer columns require int values") - return f"CAST({value} AS BIGINT)" - case StateSqlValueType.BOOLEAN: - if not isinstance(value, bool): - raise SqlValueRenderingError("state boolean columns require bool values") - return "TRUE" if value else "FALSE" - case StateSqlValueType.TIMESTAMP | StateSqlValueType.TEXT_TIMESTAMP: - if not isinstance(value, datetime): - raise SqlValueRenderingError("state timestamp columns require datetime values") - normalized: datetime = ( - value.replace(tzinfo=UTC) - if value.tzinfo is None or value.utcoffset() is None - else value.astimezone(UTC) - ) - rendered: str = _quoted(value=normalized.isoformat()) - if declared_type == StateSqlValueType.TEXT_TIMESTAMP: - return rendered - return f"CAST({rendered} AS TIMESTAMP)" - case StateSqlValueType.DATE: - if isinstance(value, datetime) or not isinstance(value, date): - raise SqlValueRenderingError("state date columns require date values") - return f"CAST({_quoted(value=value.isoformat())} AS DATE)" - case StateSqlValueType.JSON: - try: - serialized: str = json.dumps( - value, ensure_ascii=False, separators=(",", ":"), sort_keys=True - ) - except (TypeError, ValueError) as exc: - raise SqlValueRenderingError( - "state JSON columns require JSON-serializable values" - ) from exc - return _quoted(value=serialized) - - -def _quoted(*, value: str) -> str: - return "'" + value.replace("'", "''") + "'" - - -def _typed_null(*, declared_type: StateSqlValueType) -> str: - match declared_type: - case StateSqlValueType.INTEGER: - return "CAST(NULL AS BIGINT)" - case StateSqlValueType.TIMESTAMP: - return "CAST(NULL AS TIMESTAMP)" - case StateSqlValueType.DATE: - return "CAST(NULL AS DATE)" - case _: - return "NULL" diff --git a/src/sqlbuild/sql_values/main/__init__.py b/src/sqlbuild/sql_values/main/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/sql_values/main/normalize.py b/src/sqlbuild/sql_values/main/normalize.py deleted file mode 100644 index bb375ae989..0000000000 --- a/src/sqlbuild/sql_values/main/normalize.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Public entry point for typed SQL value normalization.""" - -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.sql_values.models import SqlValue, SqlValueLimits - - -def normalize_sql_value( - *, - raw_value: object, - context: str, - explicit_type: str | None = None, - limits: SqlValueLimits | None = None, -) -> SqlValue: - """Validate and normalize one authored value with contextual diagnostics.""" - - applied: SqlValueLimits = SqlValueLimits() if limits is None else limits - return cast( - SqlValue, - _native.normalize_typed_sql_value( - raw_value, - (context, explicit_type), - (applied.max_depth, applied.max_elements, applied.max_size), - ), - ) diff --git a/src/sqlbuild/sql_values/main/render_state_literal.py b/src/sqlbuild/sql_values/main/render_state_literal.py deleted file mode 100644 index 222b9177a7..0000000000 --- a/src/sqlbuild/sql_values/main/render_state_literal.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public entry point for portable typed warehouse-state SQL literals.""" - -from sqlbuild.sql_values._helpers.state_literal import ( - render_state_sql_literal as _render_state_sql_literal, -) -from sqlbuild.sql_values.types import StateSqlValueType - - -def render_state_sql_literal(*, value: object | None, declared_type: StateSqlValueType) -> str: - """Render a Python value according to its declared state-column type.""" - - return _render_state_sql_literal(value=value, declared_type=declared_type) diff --git a/src/sqlbuild/sql_values/main/validate_rendered_size.py b/src/sqlbuild/sql_values/main/validate_rendered_size.py deleted file mode 100644 index 8218fcaa3b..0000000000 --- a/src/sqlbuild/sql_values/main/validate_rendered_size.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Rendered typed SQL value size validation.""" - -from sqlbuild.sql_values.constants import DEFAULT_MAX_SQL_VALUE_SIZE -from sqlbuild.sql_values.exceptions import SqlValueValidationError - - -def validate_rendered_sql_value_size( - *, - rendered_sql: str, - context: str, - max_size: int = DEFAULT_MAX_SQL_VALUE_SIZE, -) -> None: - """Reject adapter SQL that exceeds the shared UTF-8 byte limit.""" - - rendered_size: int = len(rendered_sql.encode("utf-8")) - if rendered_size > max_size: - raise SqlValueValidationError( - f"{context} rendered value is {rendered_size} bytes; maximum is {max_size} bytes" - ) diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/test_typed_sql_rendering.py b/tests/integration/src/sqlbuild/adapters/duckdb/test_typed_sql_rendering.py index 2804239138..13886078d3 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/test_typed_sql_rendering.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/test_typed_sql_rendering.py @@ -3,8 +3,8 @@ import duckdb import pytest +from sqlbuild._native import normalize_sql_value from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.sql_values.main.normalize import normalize_sql_value from tests.integration.src.sqlbuild.adapters.duckdb._test_types import ( TypedSqlRenderingExecutionTestCase, ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py index e5df8a4dc9..9cba7be2a3 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py @@ -41,7 +41,6 @@ def test_given_plain_compile_when_running_fresh_cli_then_optional_artifact_impor ]) Path(sys.argv[2]).write_text(json.dumps({ "aggregate_models": "sqlbuild.cli.commands.models.runtime" in sys.modules, - "manifest": "sqlbuild.compiler.manifest.main.build" in sys.modules, }), encoding="utf-8") raise SystemExit(exit_code) """ @@ -58,7 +57,6 @@ def test_given_plain_compile_when_running_fresh_cli_then_optional_artifact_impor assert summary["models"] == test_case.expected_count assert json.loads(import_status_path.read_text(encoding="utf-8")) == { "aggregate_models": False, - "manifest": False, } diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 33b498ca3b..f713823e15 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -12,6 +12,7 @@ from sqlbuild._native import ( build_compile_inputs, build_compiled_project, + build_manifest, discover_project_inputs, run_compile_pipeline, semantic_shapes, @@ -26,7 +27,6 @@ ) from sqlbuild.compiler.compile.types import AuditInput, CompiledModelView, CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.sql_values.types import CollectionRendering diff --git a/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/helpers.py b/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/helpers.py index 0a29327e9d..bb8a7c17b8 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/helpers.py +++ b/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/helpers.py @@ -1,6 +1,6 @@ from __future__ import annotations -from sqlbuild.sql_values.main.normalize import normalize_sql_value +from sqlbuild._native import normalize_sql_value from sqlbuild.sql_values.models import SqlValue diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 1c3fb5caf6..047dd8951e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -9,6 +9,7 @@ import sqlbuild._native as _native from sqlbuild._native import ( + build_manifest, compiled_model_record, project_with, write_compile_target, @@ -25,7 +26,6 @@ CompiledResourceType, DiagnosticPhase, ) -from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.planner.main.execution import ( _plan_compiled_sql_test_artifacts as plan_artifacts_module, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 44dade0b57..2a730a73fc 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( assemble_project, build_compile_inputs, + build_manifest, build_native_declaration_contexts, build_scope_lookup, discover_project_inputs, @@ -61,7 +62,6 @@ ColumnTransformKind, InferredNullability, ) -from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.planner._helpers.graph.core import build_execution_upstream_deps from sqlbuild.compiler.planner._helpers.resolve.refs import resolve_ref_references from sqlbuild.compiler.scopes.models import ( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py index d427cd28b2..24bc174f74 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py @@ -6,11 +6,8 @@ import pytest -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import discover_project_inputs, validate_resource_identity from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError -from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( - validate_resource_identity, -) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( CanonicalResourceIdentityTestCase, DiscoveredResourceIdentityErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py index 9dfcd8b46e..86c2bba61f 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import build_manifest from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditRunScope, @@ -34,7 +35,6 @@ DiscoveredSourceFile, DiscoveredSqlHookFile, ) -from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.planner.models import ( AuditPlanEntry, ChainStep, diff --git a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py index bbacfc9788..9cf42aeebf 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py @@ -9,6 +9,7 @@ import sqlbuild._native as _native from sqlbuild._native import ( + build_manifest, project_with, ) from sqlbuild.compiler.compile.models import ( @@ -22,7 +23,6 @@ CompiledSourceView, ) from sqlbuild.compiler.discovery.models import DiscoveredSqlHookFile, SqlHookEntry -from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.planner.models import AuditPlanEntry, ChainStep, PlanOutput, SqlTestPlanEntry from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py index 61e6cc51c3..435c399591 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( build_scope_index, build_scope_lookup, + normalize_sql_value, query_scope_target, resolve_scope_visibility, validate_scope_index, @@ -51,7 +52,6 @@ VisibilityReason, ) from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.sql_values.main.normalize import normalize_sql_value from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( ExpectedBooleanCase, ResourceExpectation, diff --git a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py index 8286b4a231..6b68156b5c 100644 --- a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py @@ -7,11 +7,16 @@ import pytest -from sqlbuild._native import expand_sql_with_spans, map_expanded_offset, run_lint +import sqlbuild._native as native_module +from sqlbuild._native import ( + build_lint_expansion_context, + expand_sql_with_spans, + map_expanded_offset, + run_lint, +) from sqlbuild.compiler.compile.models import MappedOffset, SqlExpansionContext from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.lint._helpers import expansion -from sqlbuild.lint._helpers.expansion import build_lint_expansion_context, prepare_lint_body +from sqlbuild.lint._helpers.expansion import prepare_lint_body from sqlbuild.lint._helpers.headers import lint_file_role from sqlbuild.lint.exceptions import ProjectCompileError from sqlbuild.lint.models import LintBody, LintConfig, LintRunResult @@ -40,7 +45,7 @@ def test_given_lint_context_when_building_then_project_is_discovered_once( ) -> None: _ = (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") discovery_count: list[int] = [0] - discover: Callable[..., DiscoveredProjectInputs] = expansion.discover_project_inputs + discover: Callable[..., DiscoveredProjectInputs] = native_module.discover_project_inputs def counted_discovery( *, @@ -55,7 +60,7 @@ def counted_discovery( extract_output_column_locations=extract_output_column_locations, ) - monkeypatch.setattr(expansion, "discover_project_inputs", counted_discovery) + monkeypatch.setattr(native_module, "discover_project_inputs", counted_discovery) _ = build_lint_expansion_context(project_dir=tmp_path) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py index 81b9d91e6d..8719db8a69 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py @@ -5,9 +5,9 @@ from sqlbuild._native import ( declaration_rows, + normalize_sql_value, ) from sqlbuild.compiler.discovery.models import ConstantDeclaration -from sqlbuild.sql_values.main.normalize import normalize_sql_value from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( TypedConstantPayloadTestCase, ) diff --git a/tests/unit/src/sqlbuild/sql_values/_test_types.py b/tests/unit/src/sqlbuild/sql_values/_test_types.py index 31b6b8391c..f022acce9c 100644 --- a/tests/unit/src/sqlbuild/sql_values/_test_types.py +++ b/tests/unit/src/sqlbuild/sql_values/_test_types.py @@ -34,14 +34,6 @@ class SqlValueBehaviorTestCase: expected_identities_equal: bool = False -@dataclass(frozen=True) -class RenderedSqlValueLimitTestCase: - description: str - rendered_sql: str - max_size: int - expected_size: int - - @dataclass(frozen=True) class StateLiteralGoldenTestCase: description: str diff --git a/tests/unit/src/sqlbuild/sql_values/test_normalization.py b/tests/unit/src/sqlbuild/sql_values/test_normalization.py index f2aaf11cec..d42e56aa9b 100644 --- a/tests/unit/src/sqlbuild/sql_values/test_normalization.py +++ b/tests/unit/src/sqlbuild/sql_values/test_normalization.py @@ -5,14 +5,12 @@ import pytest +from sqlbuild._native import normalize_sql_value from sqlbuild.sql_values.exceptions import SqlValueValidationError -from sqlbuild.sql_values.main.normalize import normalize_sql_value -from sqlbuild.sql_values.main.validate_rendered_size import validate_rendered_sql_value_size from sqlbuild.sql_values.models import AuthoredSqlSet, SqlValue, SqlValueLimits from tests.unit.src.sqlbuild.sql_values._test_types import ( NormalizeSqlValueErrorTestCase, NormalizeSqlValueTestCase, - RenderedSqlValueLimitTestCase, SqlValueBehaviorTestCase, SqlValueLimitTestCase, ) @@ -150,31 +148,5 @@ def test_given_safety_limit_is_exceeded_when_normalizing_then_rejects_value( ) -@pytest.mark.parametrize( - "test_case", - [ - RenderedSqlValueLimitTestCase( - description="multibyte rendered SQL uses UTF-8 byte length", - rendered_sql="é", - max_size=1, - expected_size=2, - ) - ], - ids=lambda case: case.description, -) -def test_given_rendered_sql_exceeds_limit_when_validating_then_counts_utf8_bytes( - test_case: RenderedSqlValueLimitTestCase, -) -> None: - with pytest.raises( - SqlValueValidationError, - match=rf"rendered value is {test_case.expected_size} bytes", - ): - validate_rendered_sql_value_size( - rendered_sql=test_case.rendered_sql, - context="constant 'limited'", - max_size=test_case.max_size, - ) - - if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/sql_values/test_state_literal.py b/tests/unit/src/sqlbuild/sql_values/test_state_literal.py index a9249985fc..7f6f3a5932 100644 --- a/tests/unit/src/sqlbuild/sql_values/test_state_literal.py +++ b/tests/unit/src/sqlbuild/sql_values/test_state_literal.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal +from sqlbuild._native import render_state_sql_literal from sqlbuild.sql_values.types import StateSqlValueType from tests.unit.src.sqlbuild.sql_values._test_types import StateLiteralGoldenTestCase From 11db0876a8ed4aa397cf81b310bf328b01e1f71e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:33:54 +0100 Subject: [PATCH 35/40] refactor: classify the run lifecycle natively --- .../src/python_plan/main/mod.rs | 1 + .../src/python_plan/main/run_lifecycle.rs | 46 ++++++++++ .../sqlbuild-analysis/src/python_plan/mod.rs | 1 + .../src/python_plan/models.rs | 18 ++++ .../src/python_plan/tests.rs | 2 + .../src/python_plan/tests/helpers.rs | 16 ++++ .../python_plan/tests/test_run_lifecycle.rs | 87 +++++++++++++++++++ .../src/python_plan/tests/test_types.rs | 11 +++ .../_helpers/build_command/lifecycle.rs | 3 +- .../bindings/_helpers/check/check_command.rs | 7 +- .../compile_driver/python_plan_entries.rs | 4 +- .../src/bindings/_helpers/graph/mod.rs | 1 + .../_helpers/graph/python_lifecycle.rs | 80 +++++++++++++++++ .../bindings/_helpers/graph/registration.rs | 1 + src/sqlbuild/_native.pyi | 3 + .../python_nodes/_helpers/run_lifecycle.py | 56 ------------ .../python_nodes/main/_run_lifecycle.py | 23 ----- .../_helpers/test_python_plan_entries.py | 13 +-- .../_helpers/test_run_lifecycle.py | 6 +- .../executor/python_nodes/_helpers/helpers.py | 6 +- 20 files changed, 282 insertions(+), 103 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs create mode 100644 crates/sqlbuild-analysis/src/python_plan/models.rs create mode 100644 crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs delete mode 100644 src/sqlbuild/compiler/python_nodes/_helpers/run_lifecycle.py delete mode 100644 src/sqlbuild/compiler/python_nodes/main/_run_lifecycle.py diff --git a/crates/sqlbuild-analysis/src/python_plan/main/mod.rs b/crates/sqlbuild-analysis/src/python_plan/main/mod.rs index 8cb536efba..3c8238dc13 100644 --- a/crates/sqlbuild-analysis/src/python_plan/main/mod.rs +++ b/crates/sqlbuild-analysis/src/python_plan/main/mod.rs @@ -1,2 +1,3 @@ pub mod ordered_python_names; +pub mod run_lifecycle; pub mod upstream_closure; diff --git a/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs b/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs new file mode 100644 index 0000000000..0f33cbb1ab --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs @@ -0,0 +1,46 @@ +//! Classify selected run nodes into lifecycle-aware execution regions. + +use std::collections::{BTreeSet, HashMap}; + +use crate::python_plan::main::upstream_closure::upstream_closure; +use crate::python_plan::models::{LifecycleNode, RunLifecycle}; +use crate::python_selection::constants::{ASSET_KIND, LOADER_KIND, TASK_KIND}; + +/// Ingress is every selected warehouse loader and its selected upstream tasks, assets and loaders. +#[must_use] +pub fn run_lifecycle( + selected: &BTreeSet, + nodes: &HashMap, + upstream: &HashMap>, +) -> RunLifecycle { + let kind = |name: &str| nodes.get(name).map_or("", |node| node.kind.as_str()); + let loaders: BTreeSet = selected + .iter() + .filter(|name| { + nodes + .get(name.as_str()) + .is_some_and(|node| node.kind == LOADER_KIND && !node.external_loader) + }) + .cloned() + .collect(); + let mut ingress: BTreeSet = loaders.clone(); + for loader in &loaders { + for name in upstream_closure(loader, upstream) { + if selected.contains(&name) + && [TASK_KIND, ASSET_KIND, LOADER_KIND].contains(&kind(&name)) + { + ingress.insert(name); + } + } + } + let read_side: Vec = selected + .iter() + .filter(|name| !ingress.contains(*name) && [TASK_KIND, ASSET_KIND].contains(&kind(name))) + .cloned() + .collect(); + RunLifecycle { + ingress_names: ingress.into_iter().collect(), + ingress_loader_names: loaders.into_iter().collect(), + read_side_names: read_side, + } +} diff --git a/crates/sqlbuild-analysis/src/python_plan/mod.rs b/crates/sqlbuild-analysis/src/python_plan/mod.rs index e29b572755..1a70a48981 100644 --- a/crates/sqlbuild-analysis/src/python_plan/mod.rs +++ b/crates/sqlbuild-analysis/src/python_plan/mod.rs @@ -1,4 +1,5 @@ pub mod main; +pub mod models; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-analysis/src/python_plan/models.rs b/crates/sqlbuild-analysis/src/python_plan/models.rs new file mode 100644 index 0000000000..21b5825300 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/models.rs @@ -0,0 +1,18 @@ +//! Run lifecycle inputs and the regions selected Python nodes run in. + +/// One Python node as lifecycle classification reads it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LifecycleNode { + /// `task`, `asset`, `loader` or `check`. + pub kind: String, + /// Whether the node is a loader that owns its own warehouse connection. + pub external_loader: bool, +} + +/// Selected Python nodes split into ingress and read-side regions, each sorted. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RunLifecycle { + pub ingress_names: Vec, + pub ingress_loader_names: Vec, + pub read_side_names: Vec, +} diff --git a/crates/sqlbuild-analysis/src/python_plan/tests.rs b/crates/sqlbuild-analysis/src/python_plan/tests.rs index 4f8ba4548a..fc29fd608e 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests.rs @@ -2,5 +2,7 @@ mod helpers; #[path = "tests/test_python_plan.rs"] mod python_plan; +#[path = "tests/test_run_lifecycle.rs"] +mod run_lifecycle; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs b/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs index 55a49940b2..4489f0f16c 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs @@ -1,5 +1,7 @@ use std::collections::HashMap; +use crate::python_plan::models::LifecycleNode; + pub(crate) fn edges(rows: &[(&str, &[&str])]) -> HashMap> { rows.iter() .map(|(name, upstream)| { @@ -10,3 +12,17 @@ pub(crate) fn edges(rows: &[(&str, &[&str])]) -> HashMap> { }) .collect() } + +pub(crate) fn lifecycle_nodes(rows: &[(&str, &str, bool)]) -> HashMap { + rows.iter() + .map(|(name, kind, external_loader)| { + ( + (*name).to_owned(), + LifecycleNode { + kind: (*kind).to_owned(), + external_loader: *external_loader, + }, + ) + }) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs b/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs new file mode 100644 index 0000000000..c79fd71b91 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs @@ -0,0 +1,87 @@ +use std::collections::BTreeSet; + +use crate::python_plan::main::run_lifecycle::run_lifecycle; +use crate::python_plan::tests::helpers::{edges, lifecycle_nodes}; +use crate::python_plan::tests::test_types::LifecycleTestCase; + +#[test] +fn given_selection_when_classifying_lifecycle_then_matches_python_regions() { + let test_cases = [ + LifecycleTestCase { + description: "a warehouse loader pulls its selected upstream tasks into ingress", + selected: &["fetch_rates", "load_orders", "report", "check_orders"], + nodes: &[ + ("fetch_rates", "task", false), + ("load_orders", "loader", false), + ("report", "asset", false), + ("check_orders", "check", false), + ], + upstream: &[("load_orders", &["fetch_rates", "unselected"])], + expected_ingress: &["fetch_rates", "load_orders"], + expected_loaders: &["load_orders"], + expected_read_side: &["report"], + }, + LifecycleTestCase { + description: "external loaders are neither ingress loaders nor read-side nodes", + selected: &["sync_orders", "export"], + nodes: &[("sync_orders", "loader", true), ("export", "task", false)], + upstream: &[("export", &["sync_orders"])], + expected_ingress: &[], + expected_loaders: &[], + expected_read_side: &["export"], + }, + LifecycleTestCase { + description: "a selected external loader upstream of a warehouse loader is ingress", + selected: &["sync_orders", "load_orders"], + nodes: &[ + ("sync_orders", "loader", true), + ("load_orders", "loader", false), + ], + upstream: &[("load_orders", &["sync_orders"])], + expected_ingress: &["load_orders", "sync_orders"], + expected_loaders: &["load_orders"], + expected_read_side: &[], + }, + ]; + + for test_case in test_cases { + let selected: BTreeSet = test_case + .selected + .iter() + .map(|name| (*name).to_owned()) + .collect(); + + let lifecycle = run_lifecycle( + &selected, + &lifecycle_nodes(test_case.nodes), + &edges(test_case.upstream), + ); + + assert_eq!( + ( + lifecycle.ingress_names, + lifecycle.ingress_loader_names, + lifecycle.read_side_names, + ), + ( + test_case + .expected_ingress + .iter() + .map(|name| (*name).to_owned()) + .collect::>(), + test_case + .expected_loaders + .iter() + .map(|name| (*name).to_owned()) + .collect::>(), + test_case + .expected_read_side + .iter() + .map(|name| (*name).to_owned()) + .collect::>(), + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs b/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs index 7cab9c803f..7a4dd8f19a 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs @@ -11,3 +11,14 @@ pub(crate) struct ClosureTestCase { pub(crate) upstream: &'static [(&'static str, &'static [&'static str])], pub(crate) expected_closure: &'static [&'static str], } + +pub(crate) struct LifecycleTestCase { + pub(crate) description: &'static str, + pub(crate) selected: &'static [&'static str], + /// `(name, kind, external loader)`. + pub(crate) nodes: &'static [(&'static str, &'static str, bool)], + pub(crate) upstream: &'static [(&'static str, &'static [&'static str])], + pub(crate) expected_ingress: &'static [&'static str], + pub(crate) expected_loaders: &'static [&'static str], + pub(crate) expected_read_side: &'static [&'static str], +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index 07ddd6113d..ad9c071ce9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -18,7 +18,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; -const RUN_LIFECYCLE_MODULE: &str = "sqlbuild.compiler.python_nodes.main._run_lifecycle"; const PYTHON_NODE_MODELS: &str = "sqlbuild.compiler.python_nodes.models"; const LOADER_DAG_MODULE: &str = "sqlbuild.compiler.planner.main.selection._loader_dag"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; @@ -316,7 +315,7 @@ fn lifecycle_plan<'py>( ], )?; let plan: Bound<'py, PyAny> = call_with( - &imported(py, RUN_LIFECYCLE_MODULE, "build_python_sql_run_lifecycle")?, + &imported(py, NATIVE_MODULE, "build_python_sql_run_lifecycle")?, &[("selection", selection), ("python_graph", graph.clone())], )?; let source_type: Bound<'py, PyAny> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 4ca5e4664f..609fbd9225 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -33,7 +33,6 @@ const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const PYTHON_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; -const PYTHON_LIFECYCLE_MODULE: &str = "sqlbuild.compiler.python_nodes.main._run_lifecycle"; const PYTHON_NODE_MODELS: &str = "sqlbuild.compiler.python_nodes.models"; const PYTHON_NODE_TYPES: &str = "sqlbuild.compiler.python_nodes.types"; const EXECUTOR_MODELS: &str = "sqlbuild.executor.python_nodes.models"; @@ -337,11 +336,7 @@ fn prepare<'py>( ], )?; let lifecycle_plan: Bound<'py, PyAny> = call_with( - &imported( - py, - PYTHON_LIFECYCLE_MODULE, - "build_python_sql_run_lifecycle", - )?, + &imported(py, NATIVE_MODULE, "build_python_sql_run_lifecycle")?, &[ ("selection", selection), ("python_graph", python_graph.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs index 8e3d4733bd..de3bb1e189 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs @@ -17,7 +17,7 @@ const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; const NODE_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; -const RUN_LIFECYCLE_MODULE: &str = "sqlbuild.compiler.python_nodes.main._run_lifecycle"; +const NATIVE_MODULE: &str = "sqlbuild._native"; const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.compiler.python_nodes.models"; const PYTHON_NODE_TYPES_MODULE: &str = "sqlbuild.compiler.python_nodes.types"; const REF_TYPES_MODULE: &str = "sqlbuild.python_nodes.types"; @@ -172,7 +172,7 @@ fn build_python_run_plan_outputs<'py>( ], )?; let lifecycle: Bound<'py, PyAny> = call_with( - &imported(py, RUN_LIFECYCLE_MODULE, "build_python_sql_run_lifecycle")?, + &imported(py, NATIVE_MODULE, "build_python_sql_run_lifecycle")?, &[("selection", selection), ("python_graph", graph.clone())], )?; let entries: Bound<'py, PyTuple> = plan_entries( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs index 63db7ad9d8..e33df4195c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod algorithms; mod execution_gates; pub(crate) mod keys; pub(crate) mod project_graph; +mod python_lifecycle; pub(crate) mod python_selection; pub(crate) mod registration; pub(crate) mod views; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs new file mode 100644 index 0000000000..f6b6143dff --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs @@ -0,0 +1,80 @@ +//! Lifecycle regions of a run's selected Python nodes, classified natively. + +use std::collections::{BTreeSet, HashMap}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::python_plan::main::run_lifecycle::run_lifecycle; +use sqlbuild_analysis::python_plan::models::{LifecycleNode, RunLifecycle}; + +const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.compiler.python_nodes.models"; +const EXTERNAL_CONNECTION_MODE: &str = "external"; + +/// `build_python_sql_run_lifecycle(*, selection, python_graph)`: a `PythonSqlRunLifecyclePlan`. +#[pyfunction] +#[pyo3(signature = (*, selection, python_graph))] +fn build_python_sql_run_lifecycle<'py>( + selection: &Bound<'py, PyAny>, + python_graph: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = selection.py(); + let selected: BTreeSet = selection + .getattr("python_node_names")? + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?; + let mut nodes: HashMap = HashMap::new(); + for item in python_graph + .getattr("nodes_by_name")? + .call_method0("items")? + .try_iter()? + { + let (name, node): (String, Bound<'py, PyAny>) = item?.extract()?; + nodes.insert(name, lifecycle_node(&node)?); + } + let mut upstream: HashMap> = HashMap::new(); + for item in python_graph + .getattr("upstream_deps")? + .call_method0("items")? + .try_iter()? + { + let (name, deps): (String, Bound<'py, PyAny>) = item?.extract()?; + let deps: Vec = deps + .try_iter()? + .map(|dep| dep?.extract()) + .collect::>()?; + upstream.insert(name, deps); + } + let lifecycle: RunLifecycle = py.detach(|| run_lifecycle(&selected, &nodes, &upstream)); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item( + "ingress_python_node_names", + PyFrozenSet::new(py, lifecycle.ingress_names)?, + )?; + arguments.set_item( + "ingress_loader_names", + PyFrozenSet::new(py, lifecycle.ingress_loader_names)?, + )?; + arguments.set_item( + "read_side_python_node_names", + PyFrozenSet::new(py, lifecycle.read_side_names)?, + )?; + py.import(PYTHON_NODE_MODELS_MODULE)? + .getattr("PythonSqlRunLifecyclePlan")? + .call((), Some(&arguments)) +} + +fn lifecycle_node(node: &Bound<'_, PyAny>) -> PyResult { + let loader: Bound<'_, PyAny> = node.getattr("loader")?; + let external_loader: bool = !loader.is_none() + && loader.getattr("connection_mode")?.extract::()? == EXTERNAL_CONNECTION_MODE; + Ok(LifecycleNode { + kind: node.getattr("kind")?.extract()?, + external_loader, + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_python_sql_run_lifecycle, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs index 485560f171..f9e662f18c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs @@ -5,5 +5,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::graph::algorithms::register(module)?; crate::bindings::_helpers::graph::execution_gates::register(module)?; crate::bindings::_helpers::graph::python_selection::register(module)?; + crate::bindings::_helpers::graph::python_lifecycle::register(module)?; crate::bindings::_helpers::graph::views::register(module) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 74b78d32eb..ba66af4dd3 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2883,6 +2883,9 @@ def substitute_placeholder_defaults(sql: str, placeholders: list[tuple[str, str] def fixture_projection_facts( sql: str, dialect: str | None, / ) -> tuple[list[tuple[str, str]], list[str], list[str]]: ... +def build_python_sql_run_lifecycle( + *, selection: PythonSqlRunSelection, python_graph: PythonNodeGraph +) -> PythonSqlRunLifecyclePlan: ... def map_expanded_offset( *, offset: int, passes: Sequence[Sequence[ExpansionSpan]] ) -> MappedOffset: ... diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/run_lifecycle.py b/src/sqlbuild/compiler/python_nodes/_helpers/run_lifecycle.py deleted file mode 100644 index ecc3aa83e2..0000000000 --- a/src/sqlbuild/compiler/python_nodes/_helpers/run_lifecycle.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Lifecycle classification for build-style SQL/Python selections.""" - -from __future__ import annotations - -from sqlbuild._native import transitive_closure -from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonLoaderMetadata, - PythonNodeGraph, - PythonSqlRunLifecyclePlan, - PythonSqlRunSelection, -) -from sqlbuild.compiler.python_nodes.types import PythonNodeKind - - -def build_python_sql_run_lifecycle_plan( - *, selection: PythonSqlRunSelection, python_graph: PythonNodeGraph -) -> PythonSqlRunLifecyclePlan: - """Classify selected run nodes into lifecycle-aware execution regions.""" - - selected_python_names: frozenset[str] = selection.python_node_names - selected_loader_names: frozenset[str] = frozenset( - name - for name in selected_python_names - if python_graph.nodes_by_name[name].kind == PythonNodeKind.LOADER - if not _is_external_loader(node_name=name, python_graph=python_graph) - ) - ingress_upstream_names: set[str] = set() - for loader_name in selected_loader_names: - for upstream_name in transitive_closure( - start=loader_name, edges=python_graph.upstream_deps - ): - if upstream_name not in selected_python_names: - continue - if python_graph.nodes_by_name[upstream_name].kind in { - PythonNodeKind.TASK, - PythonNodeKind.ASSET, - PythonNodeKind.LOADER, - }: - ingress_upstream_names.add(upstream_name) - ingress_names: frozenset[str] = selected_loader_names | frozenset(ingress_upstream_names) - read_side_names: frozenset[str] = frozenset( - name - for name in selected_python_names - ingress_names - if python_graph.nodes_by_name[name].kind in {PythonNodeKind.TASK, PythonNodeKind.ASSET} - ) - return PythonSqlRunLifecyclePlan( - ingress_python_node_names=ingress_names, - ingress_loader_names=selected_loader_names, - read_side_python_node_names=read_side_names, - ) - - -def _is_external_loader(*, node_name: str, python_graph: PythonNodeGraph) -> bool: - loader: DiscoveredPythonLoaderMetadata | None = python_graph.nodes_by_name[node_name].loader - return loader is not None and loader.connection_mode == LoaderConnectionMode.EXTERNAL diff --git a/src/sqlbuild/compiler/python_nodes/main/_run_lifecycle.py b/src/sqlbuild/compiler/python_nodes/main/_run_lifecycle.py deleted file mode 100644 index c6b248a5a8..0000000000 --- a/src/sqlbuild/compiler/python_nodes/main/_run_lifecycle.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Public entrypoint for run-command Python/SQL lifecycle planning.""" - -from __future__ import annotations - -from sqlbuild.compiler.python_nodes._helpers.run_lifecycle import ( - build_python_sql_run_lifecycle_plan, -) -from sqlbuild.compiler.python_nodes.models import ( - PythonNodeGraph, - PythonSqlRunLifecyclePlan, - PythonSqlRunSelection, -) - - -def build_python_sql_run_lifecycle( - *, selection: PythonSqlRunSelection, python_graph: PythonNodeGraph -) -> PythonSqlRunLifecyclePlan: - """Classify selected run nodes into lifecycle-aware execution regions.""" - - return build_python_sql_run_lifecycle_plan( - selection=selection, - python_graph=python_graph, - ) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py index c58d2ae8dc..bc0e67322a 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py @@ -7,12 +7,13 @@ import pytest -from sqlbuild._native import build_python_plan_entries, read_latest_fingerprints +from sqlbuild._native import ( + build_python_plan_entries, + build_python_sql_run_lifecycle, + read_latest_fingerprints, +) from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.pipeline.models import PythonPlanEntry -from sqlbuild.compiler.python_nodes._helpers.run_lifecycle import ( - build_python_sql_run_lifecycle_plan, -) from sqlbuild.compiler.python_nodes.models import ( PythonNodeGraph, PythonNodeIdentity, @@ -69,7 +70,7 @@ def test_given_previous_python_identity_when_building_plan_entries_then_sets_sta ) entries: tuple[PythonPlanEntry, ...] = build_python_plan_entries( - lifecycle_plan=build_python_sql_run_lifecycle_plan( + lifecycle_plan=build_python_sql_run_lifecycle( selection=PythonSqlRunSelection( sql_keys=frozenset(), python_node_names=frozenset({"prepare_orders"}), @@ -144,7 +145,7 @@ def test_given_filtered_warehouse_fingerprints_when_planning_python_then_sets_id filtered_node_types=("model", "seed", "sql_udf"), ) entries: tuple[PythonPlanEntry, ...] = build_python_plan_entries( - lifecycle_plan=build_python_sql_run_lifecycle_plan( + lifecycle_plan=build_python_sql_run_lifecycle( selection=PythonSqlRunSelection( sql_keys=frozenset(), python_node_names=frozenset({"prepare_orders"}), diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py index 93ac2e1b3b..560bcec84c 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py @@ -4,11 +4,9 @@ import pytest +from sqlbuild._native import build_python_sql_run_lifecycle from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.python_nodes._helpers.run_lifecycle import ( - build_python_sql_run_lifecycle_plan, -) from sqlbuild.compiler.python_nodes.models import ( PythonNodeGraph, PythonSqlRunLifecyclePlan, @@ -91,7 +89,7 @@ def test_given_run_selection_when_building_lifecycle_plan_then_classifies_phases ) -> None: graph: PythonNodeGraph = build_python_node_graph_for_case(test_case.python_graph_case) - result: PythonSqlRunLifecyclePlan = build_python_sql_run_lifecycle_plan( + result: PythonSqlRunLifecyclePlan = build_python_sql_run_lifecycle( selection=test_case.selection, python_graph=graph, ) diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index 5b38105589..5647bb96ac 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import Any, ClassVar, cast +from sqlbuild._native import build_python_sql_run_lifecycle from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.assets import AssetContext, asset @@ -22,9 +23,6 @@ DiscoveredProjectInputs, DiscoveredTaskFunction, ) -from sqlbuild.compiler.python_nodes._helpers.run_lifecycle import ( - build_python_sql_run_lifecycle_plan, -) from sqlbuild.compiler.python_nodes._helpers.user_inventory import build_python_node_graph from sqlbuild.compiler.python_nodes.models import ( DiscoveredPythonNode, @@ -423,7 +421,7 @@ def assert_base_context_fields(context: BasePythonNodeContext) -> None: def build_lifecycle_plan_for_selected_python_names( *, graph: PythonNodeGraph, selected_names: frozenset[str] ) -> PythonSqlRunLifecyclePlan: - return build_python_sql_run_lifecycle_plan( + return build_python_sql_run_lifecycle( selection=PythonSqlRunSelection(sql_keys=frozenset(), python_node_names=selected_names), python_graph=graph, ) From ea4f2471099ea1c1fdff5deb2bcf2fd8748a96dd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:39:20 +0100 Subject: [PATCH 36/40] refactor: resolve rule selection natively and keep custom-rule payloads as user code --- .../src/bindings/_helpers/rules/evaluation.rs | 9 +- .../src/bindings/_helpers/rules/mod.rs | 1 + .../bindings/_helpers/rules/registration.rs | 3 +- .../src/bindings/_helpers/rules/selection.rs | 104 ++++++++++ .../src/bindings/_helpers/rules/skills.rs | 22 ++- python-boundary.toml | 5 + src/sqlbuild/_native.pyi | 9 +- .../rule_engine/_helpers/engine/catalogue.py | 14 +- .../_helpers/engine/custom_dispatch.py | 13 +- .../_helpers/engine/custom_payloads.py | 83 ++++++++ .../rule_engine/_helpers/engine/native.py | 177 ------------------ .../_helpers/guidance/skill_installation.py | 4 +- .../rule_engine/_helpers/guidance/skills.py | 4 +- .../rule_engine/_helpers/run/rules.py | 8 +- src/sqlbuild/rule_engine/main/skills.py | 4 +- 15 files changed, 238 insertions(+), 222 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs create mode 100644 src/sqlbuild/rule_engine/_helpers/engine/custom_payloads.py delete mode 100644 src/sqlbuild/rule_engine/_helpers/engine/native.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs index 30ab858b7a..29ec865df7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs @@ -2,9 +2,8 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_rules::rules::main::selected_codes; -use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; +use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::types::CompilerDetach; #[pyfunction] @@ -15,13 +14,7 @@ fn run_custom_host_json(py: Python<'_>, spec_json: &str) -> PyResult { .map_err(value_error) } -#[pyfunction] -fn selected_codes_json(request_json: &str) -> PyResult { - compiler_guard(|| selected_codes::selected_codes_json(request_json).map_err(value_error)) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(run_custom_host_json, module)?)?; - module.add_function(wrap_pyfunction!(selected_codes_json, module)?)?; crate::bindings::_helpers::rules::rows::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 0394c22076..5bad746aca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -15,5 +15,6 @@ pub(crate) mod rows; pub(crate) mod rule_cache; pub(crate) mod rules_config; pub(crate) mod run_lint; +pub(crate) mod selection; pub(crate) mod skills; pub(crate) mod sql_test_objects; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index 021431b7eb..1451384b04 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -5,7 +5,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::rules::{ builtin_catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, evaluation, lint, lint_expansion, lint_sql_bodies, lint_text, results, rule_cache, rules_config, run_lint, - skills, + selection, skills, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -21,6 +21,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { builtin_catalogue::register(module)?; rules_config::register(module)?; run_lint::register(module)?; + selection::register(module)?; results::register(module)?; skills::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs new file mode 100644 index 0000000000..f0c25105c0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs @@ -0,0 +1,104 @@ +//! The active rule codes of a rules configuration and its custom rules, resolved natively. + +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_rules::rules::main::selected_codes::selected_codes_json; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; +const RULES_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const JSON_MODULE: &str = "json"; +const BUILTINS_MODULE: &str = "builtins"; + +/// The selected built-in and custom rule codes; invalid configuration raises `RulesError`. +#[pyfunction] +#[pyo3(signature = (*, config, custom_payloads))] +fn selected_rule_codes( + config: &Bound<'_, PyAny>, + custom_payloads: &Bound<'_, PyAny>, +) -> PyResult> { + let py: Python<'_> = config.py(); + let request: Bound<'_, PyDict> = PyDict::new(py); + request.set_item( + "version", + py.import(RULES_CONSTANTS_MODULE)? + .getattr("RULES_NATIVE_API_VERSION")?, + )?; + request.set_item( + "config", + py.import(DATACLASSES_MODULE)? + .getattr("asdict")? + .call1((config,))?, + )?; + request.set_item("custom_rules", custom_payloads)?; + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("sort_keys", true)?; + arguments.set_item("default", py.import(BUILTINS_MODULE)?.getattr("str")?)?; + let encoded: String = match py + .import(JSON_MODULE)? + .getattr("dumps")? + .call((request,), Some(&arguments)) + { + Ok(encoded) => encoded.extract()?, + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + let message: String = error.value(py).str()?.extract()?; + return Err(rules_error(py, &message, Some(error))); + } + Err(error) => return Err(error), + }; + let selected: Result = compiler_guard(|| Ok(selected_codes_json(&encoded)))?; + let codes: String = selected.map_err(|message| rules_error(py, &message, None))?; + serde_json::from_str::>(&codes).map_err(|_| { + rules_error( + py, + "native rules engine returned invalid selected rule codes", + None, + ) + }) +} + +/// Whether `code` is selected and not ignored by the configuration's prefixes. +#[pyfunction] +#[pyo3(signature = (*, config, code))] +fn rule_selected(config: &Bound<'_, PyAny>, code: &str) -> PyResult { + let matches = |field: &str| -> PyResult { + for selector in config.getattr(field)?.try_iter()? { + let selector: String = selector?.extract()?; + if code.starts_with(&selector) { + return Ok(true); + } + } + Ok(false) + }; + Ok(matches("select")? && !matches("ignore")?) +} + +/// `RulesError(message)`, caused by `cause` when given. +pub(crate) fn rules_error(py: Python<'_>, message: &str, cause: Option) -> PyErr { + let raised = || -> PyResult { + let error: Bound<'_, PyAny> = py + .import(RULES_EXCEPTIONS_MODULE)? + .getattr("RulesError")? + .call1((message,))?; + Ok(PyErr::from_value(error)) + }; + let error: PyErr = raised().unwrap_or_else(|error| error); + if let Some(cause) = cause { + error.set_cause(py, Some(cause)); + } + error +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(selected_rule_codes, module)?)?; + module.add_function(wrap_pyfunction!(rule_selected, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs index bb39d0f9c2..f6a103f885 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs @@ -1,28 +1,34 @@ //! Render and check the generated rules skill. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::rules::selection::rules_error; const SKILL_OWNER: &str = "sqlbuild"; const SKILL_IDENTITY: &str = "sqlbuild-rules"; +/// Generated guidance with its ownership header; a policy failure raises `RulesError`. #[pyfunction] -fn render_owned_skill(content: &str, input_fingerprint: &str) -> PyResult { - compiler_guard(|| { - fensu_policy::render_owned_skill( +#[pyo3(signature = (*, content, input_fingerprint))] +fn render_owned_skill(py: Python<'_>, content: &str, input_fingerprint: &str) -> PyResult { + let rendered: Result = compiler_guard(|| { + Ok(fensu_policy::render_owned_skill( SKILL_OWNER, SKILL_IDENTITY, input_fingerprint, content.as_bytes(), ) - .map_err(value_error) - .and_then(|value| String::from_utf8(value).map_err(value_error)) - }) + .map_err(|error| error.to_string()) + .and_then(|value| String::from_utf8(value).map_err(|error| error.to_string()))) + })?; + rendered.map_err(|message| rules_error(py, &message, None)) } +/// Whether generated guidance is current, stale or foreign under the ownership contract. #[pyfunction] +#[pyo3(signature = (*, content, input_fingerprint))] fn skill_freshness(content: Option<&str>, input_fingerprint: &str) -> String { let freshness = fensu_policy::skill_freshness( content.map(str::as_bytes), diff --git a/python-boundary.toml b/python-boundary.toml index 104ae00f4d..e3e9f469d1 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -119,6 +119,11 @@ path = "rule_engine/_helpers/engine/custom_dispatch.py" reason = "user-code" note = "Submits custom rules to their worker thread beside native built-in rule evaluation." +[[stays]] +path = "rule_engine/_helpers/engine/custom_payloads.py" +reason = "user-code" +note = "Reads user rule functions (source lines, qualified names, import closures) to describe custom rules for native selection." + [[stays]] path = "rule_engine/_helpers/engine/custom_rules.py" reason = "user-code" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index ba66af4dd3..b2116bd81e 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1992,9 +1992,12 @@ def apply_lint_suppressions( allow_suppressions: bool, ) -> list[LintViolation]: ... def load_rules_config(*, project_dir: Path) -> RulesConfig: ... -def selected_codes_json(request_json: str) -> str: ... -def render_owned_skill(content: str, input_fingerprint: str) -> str: ... -def skill_freshness(content: str | None, input_fingerprint: str) -> str: ... +def selected_rule_codes( + *, config: RulesConfig, custom_payloads: list[dict[str, object]] +) -> list[str]: ... +def rule_selected(*, config: RulesConfig, code: str) -> bool: ... +def render_owned_skill(*, content: str, input_fingerprint: str) -> str: ... +def skill_freshness(*, content: str | None, input_fingerprint: str) -> str: ... def match_model_headers(contents: list[str]) -> list[tuple[int, int, int] | None]: ... def parse_model_headers( headers: list[str], diff --git a/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py b/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py index 852339009f..512d3f411e 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py @@ -10,12 +10,12 @@ from pathlib import Path from types import ModuleType -from sqlbuild._native import builtin_rules +from sqlbuild._native import builtin_rules, selected_rule_codes +from sqlbuild.rule_engine._helpers.engine.custom_payloads import custom_rule_payloads from sqlbuild.rule_engine._helpers.engine.definition import ( resolve_rule_signature, rule_from_value, ) -from sqlbuild.rule_engine._helpers.engine.native import native_selected_codes from sqlbuild.rule_engine.constants import ( INIT_MODULE_NAME, RULE_DECORATOR_TOKEN, @@ -77,11 +77,13 @@ def select_rules( """Resolve the active catalogue through the native Fensu rules owner.""" by_code: dict[str, Rule] = {rule.code: rule for rule in catalogue} - codes: tuple[str, ...] = native_selected_codes( + codes: list[str] = selected_rule_codes( config=config, - catalogue=catalogue, - project_dir=project_dir, - custom_payloads=custom_payloads, + custom_payloads=( + custom_rule_payloads(catalogue=catalogue, project_dir=project_dir) + if custom_payloads is None + else custom_payloads + ), ) try: return tuple(by_code[code] for code in codes) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py index 4cdbe8af63..c017406120 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py @@ -8,18 +8,15 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import rule_selected, selected_rule_codes from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity +from sqlbuild.rule_engine._helpers.engine.custom_payloads import custom_rule_payloads from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached -from sqlbuild.rule_engine._helpers.engine.native import ( - custom_rule_payloads, - finalize_native_findings, - rule_selected, - selected_codes, -) from sqlbuild.rule_engine._helpers.run.native_rows import ( build_rules_request, evaluate_rules_request, + finalize_rules_rows, ) from sqlbuild.rule_engine.constants import TYPE_PROOF_RULE_CODES from sqlbuild.rule_engine.models import ( @@ -113,7 +110,7 @@ def _evaluate_native_rows( *custom.findings, ) if not defer_suppressions: - findings = finalize_native_findings( + findings = finalize_rules_rows( config=config, project_dir=project_dir, evaluated_codes=native.selected_codes, @@ -175,6 +172,6 @@ def _selected_custom_rules( if not custom_payloads: return () selected: frozenset[str] = frozenset( - selected_codes(config=config, custom_payloads=custom_payloads) + selected_rule_codes(config=config, custom_payloads=custom_payloads) ) return tuple(rule for rule in catalogue if rule.custom and rule.code in selected) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_payloads.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_payloads.py new file mode 100644 index 0000000000..d18c5786e7 --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_payloads.py @@ -0,0 +1,83 @@ +"""Describe catalogued custom rules, reading their user functions, for native selection.""" + +from __future__ import annotations + +import inspect +from pathlib import Path + +from sqlbuild._native import ( + custom_rule_implementation_fingerprint, + custom_rule_import_closure, + custom_rule_test_evidence, +) +from sqlbuild.rule_engine.models import Rule + + +def custom_rule_payloads( + *, catalogue: tuple[Rule, ...], project_dir: Path +) -> list[dict[str, object]]: + """Describe every catalogued custom rule for native selection and evaluation.""" + + closures: dict[str, tuple[Path, ...]] = {} + return [ + _custom_rule_payload( + rule=rule, + project_dir=project_dir, + import_closure=_shared_import_closure( + rule=rule, project_dir=project_dir, cache=closures + ), + ) + for rule in catalogue + if rule.custom + ] + + +def _custom_rule_payload( + *, rule: Rule, project_dir: Path, import_closure: tuple[Path, ...] +) -> dict[str, object]: + return { + "code": rule.code, + "family": rule.family, + "slug": rule.slug, + "message": rule.message, + "remediation": rule.remediation, + "enabled_by_default": rule.enabled_by_default, + "implementation_fingerprint": custom_rule_implementation_fingerprint( + rule=rule, project_dir=project_dir, import_closure=import_closure + ), + **_custom_rule_source_payload(rule=rule, project_dir=project_dir), + "project_wide": rule.project_wide, + "check_name": getattr(rule.check, "__name__", ""), + "test_case_count": len(custom_rule_test_evidence(rule=rule, project_dir=project_dir)), + } + + +def _shared_import_closure( + *, rule: Rule, project_dir: Path, cache: dict[str, tuple[Path, ...]] +) -> tuple[Path, ...]: + if rule.source is None: + return () + source: str = str(Path(rule.source).resolve()) + if source not in cache: + cache[source] = custom_rule_import_closure(rule=rule, project_dir=project_dir) + return cache[source] + + +def _custom_rule_source_payload(*, rule: Rule, project_dir: Path) -> dict[str, object]: + source: str | None = rule.source + if source is not None: + source_path: Path = Path(source).resolve() + root: Path = project_dir.resolve() + if source_path.is_relative_to(root): + source = source_path.relative_to(root).as_posix() + try: + source_line: int = inspect.getsourcelines(rule.check)[1] + except (OSError, TypeError): + source_line = 1 + check_owner: str = getattr(rule.check, "__qualname__", getattr(rule.check, "__name__", "")) + return { + "source": source, + "source_line": source_line, + "source_column": 1, + "owner": f"{source or 'rules'}:{check_owner}", + } diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py deleted file mode 100644 index b1c6147abe..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Typed Python adapter for the private native rules engine.""" - -from __future__ import annotations - -import inspect -import json -from dataclasses import asdict -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild._native import ( - custom_rule_implementation_fingerprint, - custom_rule_import_closure, - custom_rule_test_evidence, -) -from sqlbuild.rule_engine._helpers.run.native_rows import ( - finalize_rules_rows, -) -from sqlbuild.rule_engine.constants import ( - RULES_NATIVE_API_VERSION, -) -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import ( - Finding, - Rule, - RulesConfig, -) - - -def finalize_native_findings( - *, - config: RulesConfig, - project_dir: Path, - evaluated_codes: tuple[str, ...], - findings: tuple[Finding, ...], -) -> tuple[Finding, ...]: - """Apply exception policy once to completed SQL, native, and custom findings.""" - - return finalize_rules_rows( - config=config, - project_dir=project_dir, - evaluated_codes=evaluated_codes, - findings=findings, - ) - - -def native_selected_codes( - *, - config: RulesConfig, - catalogue: tuple[Rule, ...], - project_dir: Path, - custom_payloads: list[dict[str, object]] | None = None, -) -> tuple[str, ...]: - """Resolve the active ruleset through the native Fensu adapter.""" - - return selected_codes( - config=config, - custom_payloads=( - custom_rule_payloads(catalogue=catalogue, project_dir=project_dir) - if custom_payloads is None - else custom_payloads - ), - ) - - -def selected_codes( - *, config: RulesConfig, custom_payloads: list[dict[str, object]] -) -> tuple[str, ...]: - request: dict[str, object] = { - "version": RULES_NATIVE_API_VERSION, - "config": _config_payload(config), - "custom_rules": custom_payloads, - } - try: - payload: object = json.loads( - _native.selected_codes_json(json.dumps(request, sort_keys=True, default=str)) - ) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - if not isinstance(payload, list) or any(not isinstance(code, str) for code in payload): - raise RulesError("native rules engine returned invalid selected rule codes") - return tuple(payload) - - -def render_native_owned_skill(*, content: str, input_fingerprint: str) -> str: - """Attach Fensu schema-v2 ownership to generated guidance.""" - - try: - return _native.render_owned_skill(content, input_fingerprint) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - - -def native_skill_freshness(*, content: str | None, input_fingerprint: str) -> str: - """Classify generated guidance through the Fensu ownership contract.""" - - return _native.skill_freshness(content, input_fingerprint) - - -def _config_payload(config: RulesConfig) -> dict[str, object]: - return asdict(config) - - -def rule_selected(*, config: RulesConfig, code: str) -> bool: - selected: bool = any(code.startswith(selector) for selector in config.select) - ignored: bool = any(code.startswith(selector) for selector in config.ignore) - return selected and not ignored - - -def custom_rule_payloads( - *, catalogue: tuple[Rule, ...], project_dir: Path -) -> list[dict[str, object]]: - """Describe every catalogued custom rule for native selection and evaluation.""" - - closures: dict[str, tuple[Path, ...]] = {} - return [ - _custom_rule_payload( - rule=rule, - project_dir=project_dir, - import_closure=_shared_import_closure( - rule=rule, project_dir=project_dir, cache=closures - ), - ) - for rule in catalogue - if rule.custom - ] - - -def _custom_rule_payload( - *, rule: Rule, project_dir: Path, import_closure: tuple[Path, ...] -) -> dict[str, object]: - return { - "code": rule.code, - "family": rule.family, - "slug": rule.slug, - "message": rule.message, - "remediation": rule.remediation, - "enabled_by_default": rule.enabled_by_default, - "implementation_fingerprint": custom_rule_implementation_fingerprint( - rule=rule, project_dir=project_dir, import_closure=import_closure - ), - **_custom_rule_source_payload(rule=rule, project_dir=project_dir), - "project_wide": rule.project_wide, - "check_name": getattr(rule.check, "__name__", ""), - "test_case_count": len(custom_rule_test_evidence(rule=rule, project_dir=project_dir)), - } - - -def _shared_import_closure( - *, rule: Rule, project_dir: Path, cache: dict[str, tuple[Path, ...]] -) -> tuple[Path, ...]: - if rule.source is None: - return () - source: str = str(Path(rule.source).resolve()) - if source not in cache: - cache[source] = custom_rule_import_closure(rule=rule, project_dir=project_dir) - return cache[source] - - -def _custom_rule_source_payload(*, rule: Rule, project_dir: Path) -> dict[str, object]: - source: str | None = rule.source - if source is not None: - source_path: Path = Path(source).resolve() - root: Path = project_dir.resolve() - if source_path.is_relative_to(root): - source = source_path.relative_to(root).as_posix() - try: - source_line: int = inspect.getsourcelines(rule.check)[1] - except (OSError, TypeError): - source_line = 1 - check_owner: str = getattr(rule.check, "__qualname__", getattr(rule.check, "__name__", "")) - return { - "source": source, - "source_line": source_line, - "source_column": 1, - "owner": f"{source or 'rules'}:{check_owner}", - } diff --git a/src/sqlbuild/rule_engine/_helpers/guidance/skill_installation.py b/src/sqlbuild/rule_engine/_helpers/guidance/skill_installation.py index 83cedacfeb..cbe6fa7817 100644 --- a/src/sqlbuild/rule_engine/_helpers/guidance/skill_installation.py +++ b/src/sqlbuild/rule_engine/_helpers/guidance/skill_installation.py @@ -7,7 +7,7 @@ from stat import S_IMODE from tempfile import NamedTemporaryFile -from sqlbuild.rule_engine._helpers.engine.native import native_skill_freshness +from sqlbuild._native import skill_freshness from sqlbuild.rule_engine.constants import REPLACEABLE_SKILL_STATES from sqlbuild.rule_engine.exceptions import RulesError @@ -18,7 +18,7 @@ def install_rendered_skills( """Validate and atomically replace all generated skill targets.""" freshness: tuple[str, ...] = tuple( - native_skill_freshness( + skill_freshness( content=path.read_text(encoding="utf-8") if path.is_file() else None, input_fingerprint=input_fingerprint, ) diff --git a/src/sqlbuild/rule_engine/_helpers/guidance/skills.py b/src/sqlbuild/rule_engine/_helpers/guidance/skills.py index 10a496c931..266b9ee7f3 100644 --- a/src/sqlbuild/rule_engine/_helpers/guidance/skills.py +++ b/src/sqlbuild/rule_engine/_helpers/guidance/skills.py @@ -5,7 +5,7 @@ from collections.abc import Mapping from pathlib import Path -from sqlbuild.rule_engine._helpers.engine.native import render_native_owned_skill +from sqlbuild._native import render_owned_skill from sqlbuild.rule_engine._helpers.engine.ruleset import resolve_ruleset from sqlbuild.rule_engine._helpers.guidance.thresholds import format_threshold_lines from sqlbuild.rule_engine.constants import ( @@ -104,7 +104,7 @@ def render_skills(*, config: RulesConfig, project_dir: Path) -> tuple[str, str]: body.append(f"- Replace retired `{retired}` with `{replacement}`") content: str = "\n".join(body).rstrip() + "\n" return ( - render_native_owned_skill(content=content, input_fingerprint=ruleset.fingerprint), + render_owned_skill(content=content, input_fingerprint=ruleset.fingerprint), ruleset.fingerprint, ) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index c5cf83ab5c..ccbf7171c9 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -51,14 +51,12 @@ from sqlbuild.lint.types import RelationKeys from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules from sqlbuild.rule_engine._helpers.engine.config import resolve_rule_ignore_selectors +from sqlbuild.rule_engine._helpers.engine.custom_payloads import custom_rule_payloads from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules -from sqlbuild.rule_engine._helpers.engine.native import ( - custom_rule_payloads, - finalize_native_findings, -) from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine._helpers.run.findings import group_unevaluated_findings from sqlbuild.rule_engine._helpers.run.memory import release_freed_memory +from sqlbuild.rule_engine._helpers.run.native_rows import finalize_rules_rows from sqlbuild.rule_engine.constants import SQL_RULES_CACHE_FILE, TYPE_PROOF_RULE_CODES from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main.load_config import load_rules_config @@ -197,7 +195,7 @@ def finish_rule_run( ) -> RulesRunResult: """Finalize the combined SQL, built-in and custom findings of one run.""" - finalized: tuple[Finding, ...] = finalize_native_findings( + finalized: tuple[Finding, ...] = finalize_rules_rows( config=plan.config, project_dir=plan.project_dir, evaluated_codes=tuple(rule.code for rule in plan.selected), diff --git a/src/sqlbuild/rule_engine/main/skills.py b/src/sqlbuild/rule_engine/main/skills.py index e9aed9052d..68e6c5846e 100644 --- a/src/sqlbuild/rule_engine/main/skills.py +++ b/src/sqlbuild/rule_engine/main/skills.py @@ -9,7 +9,7 @@ from filelock import FileLock -from sqlbuild.rule_engine._helpers.engine.native import native_skill_freshness +from sqlbuild._native import skill_freshness from sqlbuild.rule_engine._helpers.guidance.skill_installation import install_rendered_skills from sqlbuild.rule_engine._helpers.guidance.skills import render_skills from sqlbuild.rule_engine.constants import ( @@ -28,7 +28,7 @@ def install_skills(*, config: RulesConfig, project_dir: Path, check: bool) -> bo targets: tuple[Path, ...] = tuple(project_dir / value for value in RULES_SKILL_PATHS) if check: return all( - native_skill_freshness( + skill_freshness( content=path.read_text(encoding="utf-8") if path.is_file() else None, input_fingerprint=input_fingerprint, ) From fa1b60b7ea8ec2643ca4d487cf47e7ba69d0ff90 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:46:02 +0100 Subject: [PATCH 37/40] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index e3e9f469d1..503fd8bbd2 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 52384 +remaining_lines = 51236 [[stays]] path = "executor" From a57f19832d1a40c39014fc6231b9f7891a9e072d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:16:53 +0100 Subject: [PATCH 38/40] chore: drop a rule exception for a package without an init module --- fensu.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/fensu.toml b/fensu.toml index df00197378..36f6c10121 100644 --- a/fensu.toml +++ b/fensu.toml @@ -103,11 +103,6 @@ rule = "FFR309" path = "src/sqlbuild/compiler/resource_names/__init__.py" reason = "Resource identities are validated natively; the package keeps the error class the native check raises." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/sql_values/__init__.py" -reason = "Typed SQL values are normalized and state literals rendered natively; the package keeps the value models, types, limits and errors the native bindings build and raise." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/refactoring/__init__.py" From 82e86a39d57807c152fef826173125ec34318eeb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:57:13 +0100 Subject: [PATCH 39/40] fix: join stopped status spinner threads when the reporter closes --- .../build_progress/transient_status.rs | 40 ++++++++++++------- .../presentation/classes/_test_types.py | 6 +++ .../sqlbuild/presentation/classes/helpers.py | 21 ++++++++++ .../presentation/classes/test_status.py | 30 ++++++++++++++ 4 files changed, 83 insertions(+), 14 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs index 7b7cbcfa44..bbf3d306bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs @@ -10,7 +10,7 @@ use sqlbuild_core::progress::main::format_status_spinner_line::format_status_spi use crate::bindings::_helpers::build_progress::terminal::{ CLEAR_LINE, HIDE_CURSOR, SHOW_CURSOR, SPINNER_FRAMES, SPINNER_TICK_SECONDS, claim, flush, - release, shared_lines, start_spinner, terminal_columns, tty_flag, write, + join_spinner, release, shared_lines, start_spinner, terminal_columns, tty_flag, write, }; use crate::bindings::_helpers::compile_driver::python_calls::{imported, within}; @@ -25,6 +25,7 @@ struct StatusState { drawn: bool, frame: usize, stop_event: Option>, + threads: Vec>, exit_hook_registered: bool, } @@ -100,11 +101,26 @@ impl TransientStatusReporter { write(stream, HIDE_CURSOR)?; reporter.state().drawn = true; reporter.draw(py)?; - let (event, _thread) = start_spinner(slf.as_any())?; - reporter.state().stop_event = Some(event.unbind()); + let (event, thread) = start_spinner(slf.as_any())?; + let mut state = reporter.state(); + state.stop_event = Some(event.unbind()); + state.threads.push(thread.unbind()); Ok(()) } + fn close_status(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + reporter.locked(py, || { + release(reporter.lines.bind(py), slf.as_any())?; + reporter.exit_status(py)?; + let mut state = reporter.state(); + state.message = None; + state.active = false; + Ok(()) + }) + } + fn exit_status(&self, py: Python<'_>) -> PyResult<()> { let (drawn, event) = { let mut state = self.state(); @@ -152,7 +168,7 @@ impl TransientStatusReporter { let reporter: &Self = slf.get(); if reporter.enabled { return reporter.locked(py, || { - Self::close(slf)?; + Self::close_status(slf)?; reporter.state().message = Some(message); Self::enter_status(slf)?; claim( @@ -221,16 +237,12 @@ impl TransientStatusReporter { } fn close(slf: &Bound<'_, Self>) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let reporter: &Self = slf.get(); - reporter.locked(py, || { - release(reporter.lines.bind(py), slf.as_any())?; - reporter.exit_status(py)?; - let mut state = reporter.state(); - state.message = None; - state.active = false; - Ok(()) - }) + Self::close_status(slf)?; + let threads: Vec> = std::mem::take(&mut slf.get().state().threads); + for thread in threads { + join_spinner(thread.bind(slf.py()))?; + } + Ok(()) } /// Erase the live status so a persistent line can be written in its place. diff --git a/tests/unit/src/sqlbuild/presentation/classes/_test_types.py b/tests/unit/src/sqlbuild/presentation/classes/_test_types.py index b1b41c5a5a..89c32a2216 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/presentation/classes/_test_types.py @@ -53,3 +53,9 @@ class StatusReporterOutputTestCase: use_color: bool tty: bool expected_persistent_lines: tuple[str, ...] + + +@dataclass(frozen=True) +class StatusReporterExitTestCase: + description: str + expected_returncode: int diff --git a/tests/unit/src/sqlbuild/presentation/classes/helpers.py b/tests/unit/src/sqlbuild/presentation/classes/helpers.py index b928748d8e..ebb758f669 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/helpers.py +++ b/tests/unit/src/sqlbuild/presentation/classes/helpers.py @@ -1,6 +1,7 @@ from __future__ import annotations import io +import os import re from collections.abc import Callable from types import MappingProxyType @@ -77,3 +78,23 @@ def persistent_lines(text: str) -> tuple[str, ...]: without_cursor: str = re.sub(r"\x1b\[\?25[hl]", "", text) return tuple(line.rsplit("\x1b[K", 1)[-1] for line in without_cursor.split("\n")) + + +OPEN_SPINNER_SCRIPT: str = ( + "import io, time\n" + "from sqlbuild._native import TransientStatusReporter\n" + "class Tty(io.StringIO):\n" + " def isatty(self):\n" + " return True\n" + "reporter = TransientStatusReporter(stream=Tty())\n" + "reporter.start('Working...')\n" + "time.sleep(0.35)\n" +) + + +def without_no_progress_env() -> dict[str, str]: + """The current environment with progress spinners left enabled.""" + + environment: dict[str, str] = dict(os.environ) + environment.pop("SQLBUILD_NO_PROGRESS", None) + return environment diff --git a/tests/unit/src/sqlbuild/presentation/classes/test_status.py b/tests/unit/src/sqlbuild/presentation/classes/test_status.py index c36113fa77..9f0c1e5db5 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/test_status.py +++ b/tests/unit/src/sqlbuild/presentation/classes/test_status.py @@ -1,6 +1,8 @@ from __future__ import annotations import io +import subprocess +import sys import pytest @@ -8,13 +10,16 @@ from sqlbuild.presentation.main._progress_spinners_disabled import progress_spinners_disabled from tests.unit.src.sqlbuild.presentation.classes._test_types import ( ProgressSpinnersDisabledTestCase, + StatusReporterExitTestCase, StatusReporterOutputTestCase, ) from tests.unit.src.sqlbuild.presentation.classes.helpers import ( + OPEN_SPINNER_SCRIPT, RecordingStream, apply_no_progress_env, drive_status_reporter, persistent_lines, + without_no_progress_env, ) @@ -179,3 +184,28 @@ def test_given_status_sequence_when_reporting_then_writes_expected_persistent_li drive_status_reporter(reporter) assert persistent_lines(stream.getvalue()) == test_case.expected_persistent_lines + + +@pytest.mark.parametrize( + "test_case", + ( + StatusReporterExitTestCase( + description="an open spinner is stopped before the interpreter exits", + expected_returncode=0, + ), + ), + ids=lambda case: case.description, +) +def test_given_open_spinner_when_process_exits_then_exits_cleanly( + test_case: StatusReporterExitTestCase, +) -> None: + completed: subprocess.CompletedProcess[str] = subprocess.run( + (sys.executable, "-c", OPEN_SPINNER_SCRIPT), + capture_output=True, + text=True, + check=False, + env=without_no_progress_env(), + timeout=60, + ) + + assert completed.returncode == test_case.expected_returncode, completed.stderr From 9a1f12ead0783b973dac51dfeb257755ce0f557a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:19:34 +0100 Subject: [PATCH 40/40] chore: anchor the sql_values rule exception on an existing module --- fensu.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fensu.toml b/fensu.toml index 36f6c10121..918a9a4fd8 100644 --- a/fensu.toml +++ b/fensu.toml @@ -28,6 +28,11 @@ rule = "FFR001" path = "src/sqlbuild/cli/commands/models.py" reason = "The TYPE_CHECKING block contains annotation-only imports, avoiding executor and integration imports for read-only commands; runtime declarations remain structured models." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/sql_values/constants.py" +reason = "Typed SQL values are normalized and state literals rendered natively; the namespace package keeps the value models, types, limits and errors the native bindings build and raise." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/cursor_algebra/__init__.py"