From 9500d30b52bfa108594552064d6270e9509ba1c5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:26:46 +0100 Subject: [PATCH 01/24] feat: port sqb dbt command glue to Rust --- .../src/dispatch/_helpers/routes.rs | 2 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 5 +- .../src/dbt_command/constants.rs | 14 ++ .../dbt_command/main/dbt_output_options.rs | 22 +++ .../src/dbt_command/main/dbt_value_arg.rs | 8 + .../dbt_command/main/forwarded_dbt_args.rs | 23 +++ .../sqlbuild-core/src/dbt_command/main/mod.rs | 6 + .../dbt_command/main/render_dbt_auto_init.rs | 23 +++ .../main/render_dbt_init_preview.rs | 24 +++ .../main/render_dbt_init_summary.rs | 49 +++++ crates/sqlbuild-core/src/dbt_command/mod.rs | 8 + .../sqlbuild-core/src/dbt_command/models.rs | 28 +++ crates/sqlbuild-core/src/dbt_command/tests.rs | 6 + .../src/dbt_command/tests/helpers.rs | 19 ++ .../src/dbt_command/tests/test_dbt_command.rs | 111 +++++++++++ .../src/dbt_command/tests/test_types.rs | 15 ++ crates/sqlbuild-core/src/lib.rs | 1 + .../src/bindings/_helpers/cli/registration.rs | 7 +- .../_helpers/dbt_command/auto_init.rs | 139 ++++++++++++++ .../bindings/_helpers/dbt_command/command.rs | 175 ++++++++++++++++++ .../bindings/_helpers/dbt_command/debug.rs | 66 +++++++ .../src/bindings/_helpers/dbt_command/init.rs | 155 ++++++++++++++++ .../src/bindings/_helpers/dbt_command/mod.rs | 6 + .../_helpers/dbt_command/registration.rs | 10 + .../bindings/_helpers/dbt_command/support.rs | 56 ++++++ .../src/bindings/_helpers/mod.rs | 1 + src/sqlbuild/_native.pyi | 5 + .../cli/commands/_helpers/dbt/auto_init.py | 142 -------------- .../commands/_helpers/dbt_init/__init__.py | 1 - .../commands/_helpers/dbt_init/execution.py | 13 -- .../commands/_helpers/dbt_init/invocation.py | 58 ------ .../cli/commands/_helpers/dbt_init/outputs.py | 73 -------- src/sqlbuild/cli/commands/constants.py | 4 - src/sqlbuild/cli/commands/main/dbt/_dbt.py | 126 ------------- .../cli/commands/main/dbt/_dbt_debug.py | 51 ----- .../cli/commands/main/dbt/_dbt_init.py | 25 --- src/sqlbuild/cli/commands/models.py | 9 - ...on_args.py => _validate_execution_args.py} | 0 .../cli/commands/main/dbt/test_dbt.py | 74 ++------ .../cli/commands/main/dbt/test_init.py | 28 +-- 40 files changed, 1010 insertions(+), 578 deletions(-) create mode 100644 crates/sqlbuild-core/src/dbt_command/constants.rs create mode 100644 crates/sqlbuild-core/src/dbt_command/main/dbt_output_options.rs create mode 100644 crates/sqlbuild-core/src/dbt_command/main/dbt_value_arg.rs create mode 100644 crates/sqlbuild-core/src/dbt_command/main/forwarded_dbt_args.rs create mode 100644 crates/sqlbuild-core/src/dbt_command/main/mod.rs create mode 100644 crates/sqlbuild-core/src/dbt_command/main/render_dbt_auto_init.rs create mode 100644 crates/sqlbuild-core/src/dbt_command/main/render_dbt_init_preview.rs create mode 100644 crates/sqlbuild-core/src/dbt_command/main/render_dbt_init_summary.rs create mode 100644 crates/sqlbuild-core/src/dbt_command/mod.rs create mode 100644 crates/sqlbuild-core/src/dbt_command/models.rs create mode 100644 crates/sqlbuild-core/src/dbt_command/tests.rs create mode 100644 crates/sqlbuild-core/src/dbt_command/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/dbt_command/tests/test_dbt_command.rs create mode 100644 crates/sqlbuild-core/src/dbt_command/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_command/debug.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_command/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_command/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs delete mode 100644 src/sqlbuild/cli/commands/_helpers/dbt/auto_init.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/dbt_init/__init__.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/dbt_init/execution.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/dbt_init/invocation.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/dbt_init/outputs.py delete mode 100644 src/sqlbuild/cli/commands/main/dbt/_dbt.py delete mode 100644 src/sqlbuild/cli/commands/main/dbt/_dbt_debug.py delete mode 100644 src/sqlbuild/cli/commands/main/dbt/_dbt_init.py rename src/sqlbuild/integrations/dbt/main/cli/{validate_execution_args.py => _validate_execution_args.py} (100%) diff --git a/crates/sqlbuild-cli/src/dispatch/_helpers/routes.rs b/crates/sqlbuild-cli/src/dispatch/_helpers/routes.rs index a5278fcacf..b801524ca8 100644 --- a/crates/sqlbuild-cli/src/dispatch/_helpers/routes.rs +++ b/crates/sqlbuild-cli/src/dispatch/_helpers/routes.rs @@ -2,7 +2,7 @@ use crate::dispatch::models::{CallShape, ClassRef, Field, Route, Source}; -const DBT_IMPLEMENTATION: (&str, &str) = ("sqlbuild.cli.commands.main.dbt._dbt", "run_dbt_command"); +const DBT_IMPLEMENTATION: (&str, &str) = ("sqlbuild._native", "run_dbt_command"); pub(crate) const fn class(module: &'static str, name: &'static str) -> ClassRef { ClassRef { module, name } diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index e740334be6..725f0cea3a 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -724,10 +724,7 @@ const DBT_INIT_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const DBT_INIT_ROUTE: Route = routed( "run_dbt_init", - ( - "sqlbuild.cli.commands.main.dbt._dbt_init", - "run_dbt_init_command", - ), + ("sqlbuild._native", "run_dbt_init_command"), &DBT_INIT_ARGUMENTS, REQUEST_SHAPE, ); diff --git a/crates/sqlbuild-core/src/dbt_command/constants.rs b/crates/sqlbuild-core/src/dbt_command/constants.rs new file mode 100644 index 0000000000..e6d648a8ac --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_command/constants.rs @@ -0,0 +1,14 @@ +//! dbt interop option names and setup layout. + +pub const JSON_OPTION: &str = "--json"; +pub const VERBOSE_OPTIONS: [&str; 2] = ["--verbose", "-v"]; +pub const NO_CONNECTION_OPTION: &str = "--no-connection"; +pub const PROJECT_DIR_OPTION: &str = "--project-dir"; +pub const PROFILES_DIR_OPTION: &str = "--profiles-dir"; +pub const TARGET_OPTION: &str = "--target"; +/// Value flags whose paths are re-rooted when a twin project is created. +pub const PATH_VALUE_OPTIONS: [&str; 3] = ["--project-dir", "--profiles-dir", "--target-path"]; +/// The twin project directory created next to the dbt project. +pub const TWIN_PROJECT_DIR: &str = "sqlbuild_project"; +pub const DBT_PROJECT_FILE: &str = "dbt_project.yml"; +pub(crate) const FIELD_LABEL_WIDTH: usize = 25; diff --git a/crates/sqlbuild-core/src/dbt_command/main/dbt_output_options.rs b/crates/sqlbuild-core/src/dbt_command/main/dbt_output_options.rs new file mode 100644 index 0000000000..eb67d2687c --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_command/main/dbt_output_options.rs @@ -0,0 +1,22 @@ +//! SQLBuild's own output flags among forwarded dbt arguments. + +use crate::dbt_command::constants::{JSON_OPTION, VERBOSE_OPTIONS}; +use crate::dbt_command::models::DbtOutputOptions; + +/// Whether `--json` or a verbose flag was given, and the arguments without them. +#[must_use] +pub fn dbt_output_options(args: &[String]) -> DbtOutputOptions { + let output_option = + |arg: &String| arg == JSON_OPTION || VERBOSE_OPTIONS.contains(&arg.as_str()); + DbtOutputOptions { + json_output: args.iter().any(|arg| arg == JSON_OPTION), + verbose: args + .iter() + .any(|arg| VERBOSE_OPTIONS.contains(&arg.as_str())), + routed_args: args + .iter() + .filter(|arg| !output_option(arg)) + .cloned() + .collect(), + } +} diff --git a/crates/sqlbuild-core/src/dbt_command/main/dbt_value_arg.rs b/crates/sqlbuild-core/src/dbt_command/main/dbt_value_arg.rs new file mode 100644 index 0000000000..ffbe00683e --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_command/main/dbt_value_arg.rs @@ -0,0 +1,8 @@ +//! The value following a flag in forwarded dbt arguments. + +/// The argument after the first occurrence of `flag`, when there is one. +#[must_use] +pub fn dbt_value_arg(args: &[String], flag: &str) -> Option { + let index: usize = args.iter().position(|arg| arg == flag)?; + args.get(index + 1).cloned() +} diff --git a/crates/sqlbuild-core/src/dbt_command/main/forwarded_dbt_args.rs b/crates/sqlbuild-core/src/dbt_command/main/forwarded_dbt_args.rs new file mode 100644 index 0000000000..9002aefb1e --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_command/main/forwarded_dbt_args.rs @@ -0,0 +1,23 @@ +//! Forwarded dbt arguments with path values marked for re-rooting. + +use crate::dbt_command::constants::PATH_VALUE_OPTIONS; +use crate::dbt_command::models::ForwardedArg; + +/// Every argument, with the value after a path flag marked as a path. +#[must_use] +pub fn forwarded_dbt_args(args: &[String]) -> Vec { + let mut forwarded: Vec = Vec::new(); + let mut index: usize = 0; + while index < args.len() { + let arg: &String = &args[index]; + forwarded.push(ForwardedArg::Literal(arg.clone())); + index += 1; + if PATH_VALUE_OPTIONS.contains(&arg.as_str()) + && let Some(value) = args.get(index) + { + forwarded.push(ForwardedArg::Path(value.clone())); + index += 1; + } + } + forwarded +} diff --git a/crates/sqlbuild-core/src/dbt_command/main/mod.rs b/crates/sqlbuild-core/src/dbt_command/main/mod.rs new file mode 100644 index 0000000000..4507621a2b --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_command/main/mod.rs @@ -0,0 +1,6 @@ +pub mod dbt_output_options; +pub mod dbt_value_arg; +pub mod forwarded_dbt_args; +pub mod render_dbt_auto_init; +pub mod render_dbt_init_preview; +pub mod render_dbt_init_summary; diff --git a/crates/sqlbuild-core/src/dbt_command/main/render_dbt_auto_init.rs b/crates/sqlbuild-core/src/dbt_command/main/render_dbt_auto_init.rs new file mode 100644 index 0000000000..dd9d120375 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_command/main/render_dbt_auto_init.rs @@ -0,0 +1,23 @@ +//! The notice written when a dbt command creates its SQLBuild twin. + +use crate::cli_text::models::{CliDocument, CliStyle}; +use crate::dbt_command::constants::FIELD_LABEL_WIDTH; + +/// A blank line, the created header, the config file, and what was created. +#[must_use] +pub fn render_dbt_auto_init(project_file: &str, style: CliStyle) -> String { + let mut document: CliDocument = CliDocument::new(style); + document.header("SQLBuild dbt setup created", None); + document.lines.push(String::new()); + document.section("Setup summary"); + document.fields( + &[("Config file".to_owned(), project_file.to_owned())], + Some(FIELD_LABEL_WIDTH), + ); + document.lines.push(String::new()); + document.lines.push(format!( + "{}: a twin config that points SQLBuild at your dbt project and profile.", + style.value("What SQLBuild created") + )); + format!("\n{}", document.render(true)) +} diff --git a/crates/sqlbuild-core/src/dbt_command/main/render_dbt_init_preview.rs b/crates/sqlbuild-core/src/dbt_command/main/render_dbt_init_preview.rs new file mode 100644 index 0000000000..f9a17812d1 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_command/main/render_dbt_init_preview.rs @@ -0,0 +1,24 @@ +//! The `sqb dbt init --dry-run` preview. + +use crate::cli_text::models::{CliDocument, CliStyle}; +use crate::dbt_command::models::DbtInitSummary; +use crate::text::main::python_isspace::python_isspace; + +/// A blank line, the preview fields, and the generated configuration. +#[must_use] +pub fn render_dbt_init_preview(summary: &DbtInitSummary, style: CliStyle) -> String { + let mut document: CliDocument = CliDocument::new(style); + document.header("SQLBuild project preview", None); + document.lines.push(String::new()); + document.field("Project", (&summary.project_name, " ")); + document.field("Config file", (&summary.project_file, " ")); + document.field("Adapter", (&summary.adapter, " ")); + document.field("Target", (&summary.target_name, " ")); + document.field("Profile", (&summary.profile_name, " ")); + document.lines.push(String::new()); + document.section("Generated config"); + document + .lines + .push(summary.toml.trim_end_matches(python_isspace).to_owned()); + format!("\n{}", document.render(true)) +} diff --git a/crates/sqlbuild-core/src/dbt_command/main/render_dbt_init_summary.rs b/crates/sqlbuild-core/src/dbt_command/main/render_dbt_init_summary.rs new file mode 100644 index 0000000000..26d06a16a5 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_command/main/render_dbt_init_summary.rs @@ -0,0 +1,49 @@ +//! The summary written after `sqb dbt init` creates a twin project. + +use crate::cli_text::models::{CliDocument, CliStyle}; +use crate::dbt_command::constants::FIELD_LABEL_WIDTH; +use crate::dbt_command::models::DbtInitSummary; + +/// A blank line, the setup fields, what was created, and the next steps. +#[must_use] +pub fn render_dbt_init_summary(summary: &DbtInitSummary, style: CliStyle) -> String { + let mut document: CliDocument = CliDocument::new(style); + document.header("SQLBuild project created", None); + document.lines.push(String::new()); + document.section("Setup summary"); + let rows: Vec<(String, String)> = [ + ("Project", &summary.project_name), + ("Config file", &summary.project_file), + ("Adapter", &summary.adapter), + ("Target", &summary.target_name), + ("Profile", &summary.profile_name), + ] + .iter() + .map(|(label, value)| ((*label).to_owned(), (*value).clone())) + .collect(); + document.fields(&rows, Some(FIELD_LABEL_WIDTH)); + document.lines.push(String::new()); + document.section("What SQLBuild created"); + document.lines.push(format!( + " 1. {} ({})", + style.value("SQLBuild twin config"), + style.object_name("sqlbuild_project.toml") + )); + document.lines.push( + " This points SQLBuild at your dbt project and profile. Edit it if dbt paths" + .to_owned(), + ); + document + .lines + .push(" or profile targets need to change.".to_owned()); + document.lines.push(String::new()); + document.section("Next steps"); + document.command_line(" 1. ", (&format!("cd {}", summary.output_dir), true)); + document.lines.push(format!( + " 2. {}", + style.warning("Review the config file above.") + )); + document.command_line(" 3. ", ("sqb dbt debug", true)); + document.command_line(" 4. ", ("sqb dbt build", true)); + format!("\n{}", document.render(true)) +} diff --git a/crates/sqlbuild-core/src/dbt_command/mod.rs b/crates/sqlbuild-core/src/dbt_command/mod.rs new file mode 100644 index 0000000000..744dada3c1 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_command/mod.rs @@ -0,0 +1,8 @@ +//! `sqb dbt`: forwarded-argument handling and the dbt setup summaries. + +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/dbt_command/models.rs b/crates/sqlbuild-core/src/dbt_command/models.rs new file mode 100644 index 0000000000..79e9747cc3 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_command/models.rs @@ -0,0 +1,28 @@ +//! dbt command arguments and setup results as the command reads and reports them. + +/// The SQLBuild output flags found in forwarded dbt arguments, and the arguments without them. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtOutputOptions { + pub json_output: bool, + pub verbose: bool, + pub routed_args: Vec, +} + +/// One forwarded argument: kept as written, or a path value to re-root. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ForwardedArg { + Literal(String), + Path(String), +} + +/// A created or previewed SQLBuild twin project. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtInitSummary { + pub output_dir: String, + pub project_file: String, + pub project_name: String, + pub adapter: String, + pub target_name: String, + pub profile_name: String, + pub toml: String, +} diff --git a/crates/sqlbuild-core/src/dbt_command/tests.rs b/crates/sqlbuild-core/src/dbt_command/tests.rs new file mode 100644 index 0000000000..2ee6fd8be5 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_command/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_dbt_command.rs"] +mod dbt_command; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/dbt_command/tests/helpers.rs b/crates/sqlbuild-core/src/dbt_command/tests/helpers.rs new file mode 100644 index 0000000000..2de595dc03 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_command/tests/helpers.rs @@ -0,0 +1,19 @@ +use crate::dbt_command::models::DbtInitSummary; + +/// Owned copies of string literals. +pub(super) fn texts(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// A DuckDB twin project summary. +pub(super) fn summary() -> DbtInitSummary { + DbtInitSummary { + output_dir: "/w/sqlbuild_project".to_owned(), + project_file: "/w/sqlbuild_project/sqlbuild_project.toml".to_owned(), + project_name: "analytics".to_owned(), + adapter: "duckdb".to_owned(), + target_name: "dev".to_owned(), + profile_name: "analytics".to_owned(), + toml: "name = \"analytics\"\n\n".to_owned(), + } +} diff --git a/crates/sqlbuild-core/src/dbt_command/tests/test_dbt_command.rs b/crates/sqlbuild-core/src/dbt_command/tests/test_dbt_command.rs new file mode 100644 index 0000000000..7e13e588bc --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_command/tests/test_dbt_command.rs @@ -0,0 +1,111 @@ +use crate::cli_text::models::CliStyle; +use crate::dbt_command::main::dbt_output_options::dbt_output_options; +use crate::dbt_command::main::dbt_value_arg::dbt_value_arg; +use crate::dbt_command::main::forwarded_dbt_args::forwarded_dbt_args; +use crate::dbt_command::main::render_dbt_auto_init::render_dbt_auto_init; +use crate::dbt_command::main::render_dbt_init_preview::render_dbt_init_preview; +use crate::dbt_command::main::render_dbt_init_summary::render_dbt_init_summary; +use crate::dbt_command::models::{DbtOutputOptions, ForwardedArg}; +use crate::dbt_command::tests::helpers::{summary, texts}; +use crate::dbt_command::tests::test_types::{ArgsTestCase, RenderTestCase}; + +const PLAIN: CliStyle = CliStyle { use_color: false }; + +#[test] +fn given_forwarded_args_when_reading_them_then_splits_output_flags_and_paths() { + let test_cases = [ + ArgsTestCase { + description: "output flags are removed and path values marked", + args: texts(&[ + "--json", + "--select", + "a", + "-v", + "--profiles-dir", + "p", + "--target", + "prod", + ]), + expected_options: DbtOutputOptions { + json_output: true, + verbose: true, + routed_args: texts(&["--select", "a", "--profiles-dir", "p", "--target", "prod"]), + }, + expected_forwarded: vec![ + ForwardedArg::Literal("--json".to_owned()), + ForwardedArg::Literal("--select".to_owned()), + ForwardedArg::Literal("a".to_owned()), + ForwardedArg::Literal("-v".to_owned()), + ForwardedArg::Literal("--profiles-dir".to_owned()), + ForwardedArg::Path("p".to_owned()), + ForwardedArg::Literal("--target".to_owned()), + ForwardedArg::Literal("prod".to_owned()), + ], + expected_target: Some("prod".to_owned()), + }, + ArgsTestCase { + description: "a trailing path flag and a valueless target", + args: texts(&["--target", "--project-dir"]), + expected_options: DbtOutputOptions { + json_output: false, + verbose: false, + routed_args: texts(&["--target", "--project-dir"]), + }, + expected_forwarded: vec![ + ForwardedArg::Literal("--target".to_owned()), + ForwardedArg::Literal("--project-dir".to_owned()), + ], + expected_target: Some("--project-dir".to_owned()), + }, + ]; + + for test_case in test_cases { + assert_eq!( + dbt_output_options(&test_case.args), + test_case.expected_options, + "{}", + test_case.description + ); + assert_eq!( + forwarded_dbt_args(&test_case.args), + test_case.expected_forwarded, + "{}", + test_case.description + ); + assert_eq!( + dbt_value_arg(&test_case.args, "--target"), + test_case.expected_target, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_setup_result_when_rendering_then_writes_the_setup_documents() { + let test_cases = [ + RenderTestCase { + description: "auto-init notice", + rendered: render_dbt_auto_init("/w/sqlbuild_project.toml", PLAIN), + expected_text: "\nSQLBuild dbt setup created\n\nSetup summary:\n Config file /w/sqlbuild_project.toml\n\nWhat SQLBuild created: a twin config that points SQLBuild at your dbt project and profile.\n", + }, + RenderTestCase { + description: "dry-run preview", + rendered: render_dbt_init_preview(&summary(), PLAIN), + expected_text: "\nSQLBuild project preview\n\n Project: analytics\n Config file: /w/sqlbuild_project/sqlbuild_project.toml\n Adapter: duckdb\n Target: dev\n Profile: analytics\n\nGenerated config:\nname = \"analytics\"\n", + }, + RenderTestCase { + description: "created summary", + rendered: render_dbt_init_summary(&summary(), PLAIN), + expected_text: "\nSQLBuild project created\n\nSetup summary:\n Project analytics\n Config file /w/sqlbuild_project/sqlbuild_project.toml\n Adapter duckdb\n Target dev\n Profile analytics\n\nWhat SQLBuild created:\n 1. SQLBuild twin config (sqlbuild_project.toml)\n This points SQLBuild at your dbt project and profile. Edit it if dbt paths\n or profile targets need to change.\n\nNext steps:\n 1. cd /w/sqlbuild_project\n 2. Review the config file above.\n 3. sqb dbt debug\n 4. sqb dbt build\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_case.rendered, test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/dbt_command/tests/test_types.rs b/crates/sqlbuild-core/src/dbt_command/tests/test_types.rs new file mode 100644 index 0000000000..cddcecd96f --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_command/tests/test_types.rs @@ -0,0 +1,15 @@ +use crate::dbt_command::models::{DbtOutputOptions, ForwardedArg}; + +pub(super) struct ArgsTestCase { + pub(super) description: &'static str, + pub(super) args: Vec, + pub(super) expected_options: DbtOutputOptions, + pub(super) expected_forwarded: Vec, + pub(super) expected_target: Option, +} + +pub(super) struct RenderTestCase { + pub(super) description: &'static str, + pub(super) rendered: String, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 8e0852bd87..bc9af8ff56 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -11,6 +11,7 @@ pub mod constants; pub mod contract_command; pub mod cost_command; pub mod cost_report; +pub mod dbt_command; pub mod debug_report; pub mod format_command; pub mod freshness_text; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 7f2a199b2f..198f1e8e85 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -5,9 +5,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::cli::{argv, entry, progress, progress_reporters, progress_text}; use crate::bindings::_helpers::{ audit_command, build_command, check, clone, command_runtime, contract_command, cost_command, - cost_report, dbt_interop, debug_command, diff_report, format_command, freshness, init_command, - janitor, load, presentation_text, query_command, result_output, scenario_command, seed_command, - sql_test_command, workspace_commands, + cost_report, dbt_command, dbt_interop, debug_command, diff_report, format_command, freshness, + init_command, janitor, load, presentation_text, query_command, result_output, scenario_command, + seed_command, sql_test_command, workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -39,5 +39,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { init_command::registration::register(module)?; format_command::registration::register(module)?; cost_command::registration::register(module)?; + dbt_command::registration::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs new file mode 100644 index 0000000000..7f00d3c1dd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs @@ -0,0 +1,139 @@ +//! Create or find the SQLBuild twin project a dbt command runs from. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyString, PyTuple}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::dbt_command::constants::{ + JSON_OPTION, PROFILES_DIR_OPTION, PROJECT_DIR_OPTION, TARGET_OPTION, TWIN_PROJECT_DIR, +}; +use sqlbuild_core::dbt_command::main::dbt_value_arg::dbt_value_arg; +use sqlbuild_core::dbt_command::main::forwarded_dbt_args::forwarded_dbt_args; +use sqlbuild_core::dbt_command::main::render_dbt_auto_init::render_dbt_auto_init; +use sqlbuild_core::dbt_command::models::ForwardedArg; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::dbt_command::support::{color_enabled, flag, path, stream}; + +const DISCOVERY_CONSTANTS: &str = "sqlbuild.compiler.discovery.constants"; +const PROGRESS_REPORTER_MODULE: &str = "sqlbuild.cli.commands.classes.dbt_init_progress_reporter"; +const PROFILE_INIT_MODULE: &str = "sqlbuild.integrations.dbt.main.profile.profile_init"; +const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; + +fn project_config_found(directory: &Bound<'_, PyAny>) -> PyResult { + let name: Bound<'_, PyAny> = imported( + directory.py(), + DISCOVERY_CONSTANTS, + "PROJECT_CONFIG_FILENAME", + )?; + directory + .call_method1("__truediv__", (name,))? + .call_method0("exists")? + .is_truthy() +} + +/// `Path(raw).expanduser()`, resolved against `root` when relative. +fn resolved<'py>(root: &Bound<'py, PyAny>, raw: &str) -> PyResult> { + resolved_path(root, path(root.py(), raw)?.call_method0("expanduser")?) +} + +/// The path resolved, joined under `root` first when relative. +fn resolved_path<'py>( + root: &Bound<'py, PyAny>, + expanded: Bound<'py, PyAny>, +) -> PyResult> { + if expanded.call_method0("is_absolute")?.is_truthy()? { + return expanded.call_method0("resolve"); + } + root.call_method1("__truediv__", (expanded,))? + .call_method0("resolve") +} + +/// The arguments with path values re-rooted at the original project directory. +fn forwarded_args<'py>(args: &[String], root: &Bound<'py, PyAny>) -> PyResult> { + forwarded_dbt_args(args) + .into_iter() + .map(|arg| match arg { + ForwardedArg::Literal(text) => Ok(text), + ForwardedArg::Path(raw) => resolved(root, &raw)?.call_method0("as_posix")?.extract(), + }) + .collect() +} + +/// The project directory and arguments a dbt command runs with, creating the twin when needed. +pub(crate) fn ensure_twin_project<'py>( + effective_dir: Bound<'py, PyAny>, + args: Vec, + no_color: bool, +) -> PyResult<(Bound<'py, PyAny>, Vec)> { + let py: Python<'py> = effective_dir.py(); + if project_config_found(&effective_dir)? { + return Ok((effective_dir, args)); + } + let dbt_project_dir: Bound<'py, PyAny> = match dbt_value_arg(&args, PROJECT_DIR_OPTION) { + Some(raw) => resolved(&effective_dir, &raw)?, + None => resolved_path(&effective_dir, effective_dir.clone())?, + }; + let twin: Bound<'py, PyAny> = dbt_project_dir + .getattr("parent")? + .call_method1("__truediv__", (TWIN_PROJECT_DIR,))?; + let forwarded: Vec = forwarded_args(&args, &effective_dir)?; + if project_config_found(&twin)? { + return Ok((twin, forwarded)); + } + let use_color: bool = + !no_color && !args.iter().any(|arg| arg == JSON_OPTION) && color_enabled(py)?; + let progress_stream: Bound<'py, PyAny> = stream(py, "stderr")?; + let progress: Bound<'py, PyAny> = call_with( + &imported(py, PROGRESS_REPORTER_MODULE, "DbtInitProgressReporter")?, + &[ + ("stream", progress_stream.clone()), + ("use_color", flag(py, use_color)), + ], + )?; + let callbacks: Bound<'py, PyAny> = call_with( + &imported(py, DBT_MODELS, "DbtInitProgressCallbacks")?, + &[ + ("start", progress.getattr("start")?), + ("complete", progress.getattr("complete")?), + ], + )?; + let none: Bound<'py, PyAny> = py.None().into_bound(py); + let profiles_dir: Bound<'py, PyAny> = match dbt_value_arg(&args, PROFILES_DIR_OPTION) { + Some(raw) => path(py, &raw)?, + None => none.clone(), + }; + let target_name: Bound<'py, PyAny> = match dbt_value_arg(&args, TARGET_OPTION) { + Some(raw) => PyString::new(py, &raw).into_any(), + None => none.clone(), + }; + let request: Bound<'py, PyAny> = call_with( + &imported(py, DBT_MODELS, "DbtInitRequest")?, + &[ + ("cwd", effective_dir), + ("dbt_project_dir", dbt_project_dir), + ("profiles_dir", profiles_dir), + ("profile_name", none.clone()), + ("target_name", target_name), + ("sqb_output_dir", twin), + ("dry_run", flag(py, false)), + ("overwrite", flag(py, false)), + ("skip_dbt_debug", flag(py, true)), + ("progress_callbacks", callbacks), + ], + )?; + let result: Bound<'py, PyAny> = call_with( + &imported(py, PROFILE_INIT_MODULE, "run_dbt_profile_init")?, + &[("request", request)], + )?; + let project_file: String = result.getattr("project_file")?.str()?.to_string(); + progress_stream.call_method1( + "write", + (render_dbt_auto_init(&project_file, CliStyle { use_color }),), + )?; + Ok((result.getattr("output_dir")?, forwarded)) +} + +/// The arguments as a Python tuple. +pub(crate) fn args_tuple<'py>(py: Python<'py>, args: &[String]) -> PyResult> { + Ok(PyTuple::new(py, args)?.into_any()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs new file mode 100644 index 0000000000..eef0e1eae4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs @@ -0,0 +1,175 @@ +//! `sqb dbt plan|run|build|debug`: dbt interop through the SQLBuild twin project. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyInt}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::dbt_command::main::dbt_output_options::dbt_output_options; +use sqlbuild_core::dbt_command::models::DbtOutputOptions; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::dbt_command::auto_init::{args_tuple, ensure_twin_project}; +use crate::bindings::_helpers::dbt_command::debug::run_dbt_debug; +use crate::bindings::_helpers::dbt_command::support::{color_enabled, effective_dir, flag, stream}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const VALIDATE_ARGS_MODULE: &str = "sqlbuild.integrations.dbt.main.cli._validate_execution_args"; +const PLAN_MODULE: &str = "sqlbuild.integrations.dbt.main.pipeline.plan"; +const EXECUTE_MODULE: &str = "sqlbuild.integrations.dbt.main.pipeline.execute"; +const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; +const PRESENTATION_MODELS: &str = "sqlbuild.presentation.models"; +const PLAN_COMMAND: &str = "plan"; +const DEBUG_COMMAND: &str = "debug"; +const BOUNDED_SECTION_ENTRIES: i64 = 10; + +/// Progress goes to stderr under `--json` so stdout stays machine-readable. +struct Progress<'py> { + stream: Bound<'py, PyAny>, + use_color: bool, + reporter: Bound<'py, PyAny>, +} + +impl<'py> Progress<'py> { + fn new(py: Python<'py>, options: &DbtOutputOptions, no_color: bool) -> PyResult { + let use_color: bool = !no_color && !options.json_output && color_enabled(py)?; + let stream: Bound<'py, PyAny> = stream( + py, + if options.json_output { + "stderr" + } else { + "stdout" + }, + )?; + let reporter: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "PlanningProgressReporter")?, + &[ + ("stream", stream.clone()), + ("use_color", flag(py, use_color)), + ], + )?; + Ok(Self { + stream, + use_color, + reporter, + }) + } + + fn blank_line(&self) -> PyResult<()> { + self.stream.call_method1("write", ("\n",))?; + self.stream.call_method0("flush")?; + Ok(()) + } +} + +/// `sqb dbt plan`: the combined dbt and SQLBuild plan, as text or JSON. +fn run_plan(project_dir: &Bound<'_, PyAny>, args: &[String], no_color: bool) -> PyResult { + let py: Python<'_> = project_dir.py(); + let options: DbtOutputOptions = dbt_output_options(args); + let progress: Progress<'_> = Progress::new(py, &options, no_color)?; + if !options.json_output { + progress.blank_line()?; + } + let plan: Bound<'_, PyAny> = call_with( + &imported(py, PLAN_MODULE, "plan_dbt_interop_from_project")?, + &[ + ("project_dir", project_dir.clone()), + ("args", args_tuple(py, &options.routed_args)?), + ("on_progress", progress.reporter.getattr("on_progress")?), + ("progress_stream", progress.stream.clone()), + ("use_color", flag(py, progress.use_color)), + ], + )?; + let display: Bound<'_, PyAny> = call_with( + &imported(py, PRESENTATION_MODELS, "DisplayOptions")?, + &[( + "max_entries_per_section", + if options.verbose { + py.None().into_bound(py) + } else { + PyInt::new(py, BOUNDED_SECTION_ENTRIES).into_any() + }, + )], + )?; + let rendered: String = call_with( + &imported(py, NATIVE_MODULE, "render_dbt_interop_plan")?, + &[ + ("plan", plan), + ("json_output", flag(py, options.json_output)), + ("use_color", flag(py, progress.use_color)), + ("display_options", display), + ], + )? + .extract()?; + stream(py, "stdout")?.call_method1("write", (format!("{rendered}\n"),))?; + Ok(0) +} + +/// `sqb dbt run|build`: dbt first, then the SQLBuild work the combined graph selects. +fn run_execution( + command: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, + args: &[String], + no_color: bool, +) -> PyResult { + let py: Python<'_> = project_dir.py(); + let options: DbtOutputOptions = dbt_output_options(args); + let progress: Progress<'_> = Progress::new(py, &options, no_color)?; + progress.blank_line()?; + let request: Bound<'_, PyAny> = call_with( + &imported(py, DBT_MODELS, "DbtInteropExecutionRequest")?, + &[ + ("command", command.clone()), + ("project_dir", project_dir.clone()), + ("args", args_tuple(py, &options.routed_args)?), + ("on_progress", progress.reporter.getattr("on_progress")?), + ("progress_stream", progress.stream.clone()), + ("dbt_stdout_stream", stream(py, "stdout")?), + ("use_color", flag(py, progress.use_color)), + ("verbose", flag(py, options.verbose)), + ("json_output", flag(py, options.json_output)), + ], + )?; + imported(py, EXECUTE_MODULE, "execute_dbt_interop_from_project")? + .call1((request,))? + .extract() +} + +/// Execute one `sqb dbt` interop command. +#[pyfunction] +#[pyo3(signature = (**options))] +fn run_dbt_command(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "run_dbt_command", + options, + (&["command", "project_dir", "args", "no_color"], &[]), + )?; + let command: Bound<'_, PyAny> = arguments.value("command")?; + let args_value: Bound<'_, PyAny> = arguments.value("args")?; + let no_color: bool = arguments.flag("no_color")?; + call_with( + &imported(py, VALIDATE_ARGS_MODULE, "validate_dbt_execution_args")?, + &[("command", command.clone()), ("args", args_value.clone())], + )?; + let args: Vec = args_value + .try_iter()? + .map(|arg| arg?.extract()) + .collect::>()?; + let (project_dir, args) = ensure_twin_project( + effective_dir(py, Some(arguments.value("project_dir")?))?, + args, + no_color, + )?; + let command_name: String = command.getattr("value")?.extract()?; + match command_name.as_str() { + PLAN_COMMAND => run_plan(&project_dir, &args, no_color), + DEBUG_COMMAND => run_dbt_debug(&project_dir, &args, no_color), + _ => run_execution(&command, &project_dir, &args, no_color), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_dbt_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/debug.rs new file mode 100644 index 0000000000..d2b5733349 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/debug.rs @@ -0,0 +1,66 @@ +//! `sqb dbt debug`: dbt's own diagnostics, then SQLBuild's. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_core::dbt_command::constants::NO_CONNECTION_OPTION; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::dbt_command::auto_init::args_tuple; +use crate::bindings::_helpers::dbt_command::support::{color_enabled, flag, status, stream}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const DEBUG_MODULE: &str = "sqlbuild.integrations.dbt.main.pipeline.debug"; + +/// Run `body` under a stderr status line that is always closed; returns its exit code. +fn with_status( + py: Python<'_>, + (message, use_color): (&str, bool), + body: impl FnOnce() -> PyResult, +) -> PyResult { + let reporter: Bound<'_, PyAny> = status(&stream(py, "stderr")?, use_color)?; + reporter.call_method1("start", (message,))?; + let outcome: PyResult = body(); + let closed: PyResult> = reporter.call_method0("close"); + let code: i64 = outcome?; + closed?; + Ok(code) +} + +/// Run dbt debug, then SQLBuild diagnostics; 0 only when both pass. +pub(crate) fn run_dbt_debug( + project_dir: &Bound<'_, PyAny>, + args: &[String], + no_color: bool, +) -> PyResult { + let py: Python<'_> = project_dir.py(); + let use_color: bool = !no_color && color_enabled(py)?; + let dbt_code: i64 = with_status(py, ("Running dbt debug...", use_color), || { + call_with( + &imported(py, DEBUG_MODULE, "debug_dbt_from_project")?, + &[ + ("project_dir", project_dir.clone()), + ("args", args_tuple(py, args)?), + ("stdout_stream", stream(py, "stdout")?), + ("stderr_stream", stream(py, "stderr")?), + ], + )? + .extract() + })?; + stream(py, "stdout")?.call_method1("write", ("\n",))?; + let sqlbuild_code: i64 = + with_status(py, ("Running SQLBuild diagnostics...", use_color), || { + call_with( + &imported(py, NATIVE_MODULE, "run_debug_command")?, + &[ + ("project_dir", project_dir.clone()), + ("no_color", flag(py, no_color)), + ( + "no_connection", + flag(py, args.iter().any(|arg| arg == NO_CONNECTION_OPTION)), + ), + ("json_output", flag(py, false)), + ], + )? + .extract() + })?; + Ok(i32::from(dbt_code != 0 || sqlbuild_code != 0)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs new file mode 100644 index 0000000000..32b16cc2e4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs @@ -0,0 +1,155 @@ +//! `sqb dbt init`: create or preview a SQLBuild twin project for a dbt project. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::dbt_command::constants::DBT_PROJECT_FILE; +use sqlbuild_core::dbt_command::main::render_dbt_init_preview::render_dbt_init_preview; +use sqlbuild_core::dbt_command::main::render_dbt_init_summary::render_dbt_init_summary; +use sqlbuild_core::dbt_command::models::DbtInitSummary; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::dbt_command::support::{color_enabled, flag, path, stream}; + +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const PROGRESS_REPORTER_MODULE: &str = "sqlbuild.cli.commands.classes.dbt_init_progress_reporter"; +const PROFILE_INIT_MODULE: &str = "sqlbuild.integrations.dbt.main.profile.profile_init"; +const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; + +fn missing_project(py: Python<'_>) -> PyErr { + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", "C242")?; + imported(py, ERRORS_MODULE, "CliUserError")?.call( + ("sqb dbt init requires --project-dir pointing at a dbt project",), + Some(&options), + ) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// `Path(value)`, or `None` when the option was not given. +fn optional_path<'py>(value: Bound<'py, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(value); + } + imported(value.py(), "pathlib", "Path")?.call1((value,)) +} + +/// The dbt project directory: the option, else `.` when the working directory is a dbt project. +fn dbt_project_dir<'py>(request: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = request.py(); + let given: Bound<'py, PyAny> = request.getattr("dbt_project_dir")?; + if !given.is_none() { + return optional_path(given); + } + let in_project: bool = request + .getattr("cwd")? + .call_method1("__truediv__", (DBT_PROJECT_FILE,))? + .call_method0("exists")? + .is_truthy()?; + if in_project { + return path(py, "."); + } + Err(missing_project(py)) +} + +fn text(result: &Bound<'_, PyAny>, name: &str) -> PyResult { + Ok(result.getattr(name)?.str()?.to_string()) +} + +fn summary(result: &Bound<'_, PyAny>) -> PyResult { + Ok(DbtInitSummary { + output_dir: text(result, "output_dir")?, + project_file: text(result, "project_file")?, + project_name: text(result, "project_name")?, + adapter: text(result, "adapter")?, + target_name: text(result, "target_name")?, + profile_name: text(result, "profile_name")?, + toml: text(result, "toml")?, + }) +} + +/// Execute SQLBuild-owned `sqb dbt init`. +#[pyfunction] +#[pyo3(signature = (request))] +fn run_dbt_init_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let project_dir: Bound<'_, PyAny> = dbt_project_dir(request)?; + let use_color: bool = color_enabled(py)?; + let progress: Bound<'_, PyAny> = call_with( + &imported(py, PROGRESS_REPORTER_MODULE, "DbtInitProgressReporter")?, + &[ + ("stream", stream(py, "stdout")?), + ("use_color", flag(py, use_color)), + ], + )?; + let callbacks: Bound<'_, PyAny> = call_with( + &imported(py, DBT_MODELS, "DbtInitProgressCallbacks")?, + &[ + ("start", progress.getattr("start")?), + ("complete", progress.getattr("complete")?), + ], + )?; + let init_request: Bound<'_, PyAny> = call_with( + &imported(py, DBT_MODELS, "DbtInitRequest")?, + &[ + ("cwd", request.getattr("cwd")?), + ("dbt_project_dir", project_dir), + ( + "profiles_dir", + optional_path(request.getattr("profiles_dir")?)?, + ), + ("profile_name", request.getattr("profile_name")?), + ("target_name", request.getattr("target_name")?), + ( + "sqb_output_dir", + optional_path(request.getattr("sqb_output_dir")?)?, + ), + ("dry_run", request.getattr("dry_run")?), + ("overwrite", request.getattr("overwrite")?), + ("skip_dbt_debug", request.getattr("skip_dbt_debug")?), + ("progress_callbacks", callbacks), + ], + )?; + call_with( + &imported( + py, + PROFILE_INIT_MODULE, + "_validate_dbt_profile_init_request", + )?, + &[("request", init_request.clone())], + )?; + let result: Bound<'_, PyAny> = call_with( + &imported(py, PROFILE_INIT_MODULE, "run_dbt_profile_init")?, + &[("request", init_request)], + )?; + let style: CliStyle = CliStyle { use_color }; + let stdout: Bound<'_, PyAny> = stream(py, "stdout")?; + if request.getattr("dry_run")?.is_truthy()? { + stdout.call_method1( + "write", + (render_dbt_init_preview(&summary(&result)?, style),), + )?; + return Ok(0); + } + stdout.call_method1( + "write", + (render_dbt_init_summary(&summary(&result)?, style),), + )?; + for warning in result.getattr("warnings")?.try_iter()? { + stream(py, "stderr")? + .call_method1("write", (format!("Warning: {}\n", warning?.str()?),))?; + } + Ok(0) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_dbt_init_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/mod.rs new file mode 100644 index 0000000000..05deced57e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod auto_init; +pub(crate) mod command; +pub(crate) mod debug; +pub(crate) mod init; +pub(crate) mod registration; +pub(crate) mod support; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/registration.rs new file mode 100644 index 0000000000..88cd8e8b34 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/registration.rs @@ -0,0 +1,10 @@ +//! Register the native `sqb dbt` commands. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::dbt_command::{command, init}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + command::register(module)?; + init::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs new file mode 100644 index 0000000000..01348c309c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs @@ -0,0 +1,56 @@ +//! Shared Python lookups of the dbt commands: streams, colour, paths and status lines. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyBool; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; + +/// `sys.stdout` or `sys.stderr`, looked up when written. +pub(crate) fn stream<'py>(py: Python<'py>, name: &str) -> PyResult> { + py.import("sys")?.getattr(name) +} + +/// `supports_color()`. +pub(crate) fn color_enabled(py: Python<'_>) -> PyResult { + imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .is_truthy() +} + +pub(crate) fn flag(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { + PyBool::new(py, value).to_owned().into_any() +} + +/// The project directory, or the working directory when none was given. +pub(crate) fn effective_dir<'py>( + py: Python<'py>, + project_dir: Option>, +) -> PyResult> { + match project_dir.filter(|path| !path.is_none()) { + Some(path) => Ok(path), + None => imported(py, "pathlib", "Path")?.call_method0("cwd"), + } +} + +/// `pathlib.Path(text)`. +pub(crate) fn path<'py>(py: Python<'py>, text: &str) -> PyResult> { + imported(py, "pathlib", "Path")?.call1((text,)) +} + +/// A transient status line on `stream`. +pub(crate) fn status<'py>( + stream: &Bound<'py, PyAny>, + use_color: bool, +) -> PyResult> { + let py: Python<'py> = stream.py(); + call_with( + &imported(py, STATUS_REPORTER_MODULE, "TransientStatusReporter")?, + &[ + ("stream", stream.clone()), + ("use_color", flag(py, use_color)), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index d44788258f..09ed288c03 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -22,6 +22,7 @@ pub(crate) mod cost_command; pub(crate) mod cost_report; pub(crate) mod cursor; pub(crate) mod dag_artifact; +pub(crate) mod dbt_command; pub(crate) mod dbt_interop; pub(crate) mod debug_command; pub(crate) mod diagnostics; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index f457147d0d..e7fa81c4b1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -30,6 +30,7 @@ from sqlbuild.cli.commands.models import ( CloneCommandRequest, ContractCommandRequest, CostCommandRequest, + DbtInitCommandRequest, DiffCommandRequest, FreshnessCommandRequest, JanitorCommandRequest, @@ -2940,6 +2941,10 @@ def run_audit_command(*, request: AuditCommandRequest) -> int: ... def run_contract_command(*, request: ContractCommandRequest) -> int: ... def run_init_command(project_dir: Path | None = None) -> int: ... def run_cost_command(request: CostCommandRequest) -> int: ... +def run_dbt_command( + *, command: DbtInteropCommand, project_dir: Path | None, args: tuple[str, ...], no_color: bool +) -> int: ... +def run_dbt_init_command(request: DbtInitCommandRequest) -> int: ... def refresh_pending_cost_run(request: CostCommandRequest) -> None: ... def parse_cost_since_bound(*, value: str | None) -> datetime | None: ... def run_query_command( diff --git a/src/sqlbuild/cli/commands/_helpers/dbt/auto_init.py b/src/sqlbuild/cli/commands/_helpers/dbt/auto_init.py deleted file mode 100644 index b05f7dd5c9..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/dbt/auto_init.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Auto-init support for dbt interop commands.""" - -from __future__ import annotations - -import sys -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import CliDocument -from sqlbuild.cli.commands.classes.dbt_init_progress_reporter import DbtInitProgressReporter -from sqlbuild.compiler.discovery.constants import PROJECT_CONFIG_FILENAME -from sqlbuild.integrations.dbt.main.profile.profile_init import run_dbt_profile_init -from sqlbuild.integrations.dbt.models import DbtInitProgressCallbacks, DbtInitRequest, DbtInitResult -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.supports_color import supports_color - -_JSON_OUTPUT_OPTION: str = "--json" - - -def ensure_sqlbuild_project_for_dbt_command( - *, project_dir: Path | None, args: tuple[str, ...], no_color: bool -) -> tuple[Path, tuple[str, ...]]: - """Create or resolve the SQLBuild twin project for a dbt interop command.""" - - effective_project_dir: Path = project_dir if project_dir is not None else Path.cwd() - if (effective_project_dir / PROJECT_CONFIG_FILENAME).exists(): - return effective_project_dir, args - - dbt_project_dir: Path = _resolve_dbt_project_dir( - effective_project_dir=effective_project_dir, - args=args, - ) - twin_project_dir: Path = dbt_project_dir.parent / "sqlbuild_project" - forwarded_args: tuple[str, ...] = _normalize_forwarded_path_args( - args=args, - original_project_dir=effective_project_dir, - ) - if (twin_project_dir / PROJECT_CONFIG_FILENAME).exists(): - return twin_project_dir, forwarded_args - - use_color: bool = not no_color and _JSON_OUTPUT_OPTION not in args and supports_color() - progress_stream: TextIO = sys.stderr - progress: DbtInitProgressReporter = DbtInitProgressReporter( - stream=progress_stream, - use_color=use_color, - ) - result: DbtInitResult = run_dbt_profile_init( - request=DbtInitRequest( - cwd=effective_project_dir, - dbt_project_dir=dbt_project_dir, - profiles_dir=_optional_path_arg(args=args, flag="--profiles-dir"), - profile_name=None, - target_name=_value_arg(args=args, flag="--target"), - sqb_output_dir=twin_project_dir, - dry_run=False, - overwrite=False, - skip_dbt_debug=True, - progress_callbacks=DbtInitProgressCallbacks( - start=progress.start, - complete=progress.complete, - ), - ) - ) - _render_auto_init_result(result=result, stream=progress_stream, use_color=use_color) - return result.output_dir, forwarded_args - - -def _render_auto_init_result(*, result: DbtInitResult, stream: TextIO, use_color: bool) -> None: - style: CliStyle = CliStyle(use_color=use_color) - doc: CliDocument = CliDocument(style) - doc.header(text="SQLBuild dbt setup created") - doc.blank() - doc.section("Setup summary") - doc.fields( - rows=(("Config file", str(result.project_file)),), - label_width=25, - ) - doc.blank() - doc.line( - f"{style.value('What SQLBuild created')}: a twin config that points SQLBuild " - "at your dbt project and profile." - ) - stream.write("\n" + doc.render()) - - -def _resolve_dbt_project_dir(*, effective_project_dir: Path, args: tuple[str, ...]) -> Path: - raw_project_dir: str | None = _value_arg(args=args, flag="--project-dir") - dbt_project_dir: Path = ( - Path(raw_project_dir).expanduser() if raw_project_dir is not None else effective_project_dir - ) - if dbt_project_dir.is_absolute(): - return dbt_project_dir.resolve() - return (effective_project_dir / dbt_project_dir).resolve() - - -def _optional_path_arg(*, args: tuple[str, ...], flag: str) -> Path | None: - raw_value: str | None = _value_arg(args=args, flag=flag) - if raw_value is None: - return None - return Path(raw_value) - - -def _normalize_forwarded_path_args( - *, args: tuple[str, ...], original_project_dir: Path -) -> tuple[str, ...]: - value_path_flags: frozenset[str] = frozenset( - {"--project-dir", "--profiles-dir", "--target-path"} - ) - normalized: list[str] = [] - skip_next: bool = False - index: int - arg: str - for index, arg in enumerate(args): - if skip_next: - skip_next = False - continue - normalized.append(arg) - if arg not in value_path_flags: - continue - if index + 1 >= len(args): - continue - normalized.append( - _resolve_forwarded_path_arg(raw_value=args[index + 1], root=original_project_dir) - ) - skip_next = True - return tuple(normalized) - - -def _resolve_forwarded_path_arg(*, raw_value: str, root: Path) -> str: - path: Path = Path(raw_value).expanduser() - if path.is_absolute(): - return path.resolve().as_posix() - return (root / path).resolve().as_posix() - - -def _value_arg(*, args: tuple[str, ...], flag: str) -> str | None: - if flag not in args: - return None - index: int = args.index(flag) - if index + 1 >= len(args): - return None - return args[index + 1] diff --git a/src/sqlbuild/cli/commands/_helpers/dbt_init/__init__.py b/src/sqlbuild/cli/commands/_helpers/dbt_init/__init__.py deleted file mode 100644 index 18bff47af7..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/dbt_init/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Helpers for SQLBuild-owned dbt init command.""" diff --git a/src/sqlbuild/cli/commands/_helpers/dbt_init/execution.py b/src/sqlbuild/cli/commands/_helpers/dbt_init/execution.py deleted file mode 100644 index 0697ac8bfb..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/dbt_init/execution.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Execution phase for the dbt init command.""" - -from __future__ import annotations - -from sqlbuild.cli.commands.models import DbtInitInvocation -from sqlbuild.integrations.dbt.main.profile.profile_init import run_dbt_profile_init -from sqlbuild.integrations.dbt.models import DbtInitResult - - -def execute_dbt_init(invocation: DbtInitInvocation) -> DbtInitResult: - """Run dbt profile initialization.""" - - return run_dbt_profile_init(request=invocation.request) diff --git a/src/sqlbuild/cli/commands/_helpers/dbt_init/invocation.py b/src/sqlbuild/cli/commands/_helpers/dbt_init/invocation.py deleted file mode 100644 index 5ef8585b93..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/dbt_init/invocation.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Invocation resolution for the dbt init command.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -from sqlbuild.cli.commands.classes.dbt_init_progress_reporter import DbtInitProgressReporter -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ( - DbtInitCommandRequest, - DbtInitInvocation, -) -from sqlbuild.integrations.dbt.main.profile.profile_init import _validate_dbt_profile_init_request -from sqlbuild.integrations.dbt.models import DbtInitProgressCallbacks, DbtInitRequest -from sqlbuild.presentation.main.supports_color import supports_color - - -def resolve_dbt_init_invocation(*, request: DbtInitCommandRequest) -> DbtInitInvocation: - """Resolve paths, validation, and progress callbacks for dbt init.""" - - resolved_dbt_project_dir: Path = _resolve_dbt_project_dir(request=request) - use_color: bool = supports_color() - progress: DbtInitProgressReporter = DbtInitProgressReporter( - stream=sys.stdout, - use_color=use_color, - ) - init_request: DbtInitRequest = DbtInitRequest( - cwd=request.cwd, - dbt_project_dir=resolved_dbt_project_dir, - profiles_dir=None if request.profiles_dir is None else Path(request.profiles_dir), - profile_name=request.profile_name, - target_name=request.target_name, - sqb_output_dir=None if request.sqb_output_dir is None else Path(request.sqb_output_dir), - dry_run=request.dry_run, - overwrite=request.overwrite, - skip_dbt_debug=request.skip_dbt_debug, - progress_callbacks=DbtInitProgressCallbacks( - start=progress.start, - complete=progress.complete, - ), - ) - _validate_dbt_profile_init_request(request=init_request) - return DbtInitInvocation(request=init_request, use_color=use_color) - - -def _resolve_dbt_project_dir(*, request: DbtInitCommandRequest) -> Path: - resolved_dbt_project_dir: Path | None = ( - None if request.dbt_project_dir is None else Path(request.dbt_project_dir) - ) - if resolved_dbt_project_dir is None and (request.cwd / "dbt_project.yml").exists(): - resolved_dbt_project_dir = Path(".") - if resolved_dbt_project_dir is None: - raise CliUserError( - "sqb dbt init requires --project-dir pointing at a dbt project", - code="C242", - ) - return resolved_dbt_project_dir diff --git a/src/sqlbuild/cli/commands/_helpers/dbt_init/outputs.py b/src/sqlbuild/cli/commands/_helpers/dbt_init/outputs.py deleted file mode 100644 index 4c4f1b2045..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/dbt_init/outputs.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Output rendering for the dbt init command.""" - -from __future__ import annotations - -import sys - -from sqlbuild._native import CliDocument -from sqlbuild.cli.commands.models import DbtInitCommandRequest -from sqlbuild.integrations.dbt.models import DbtInitResult -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def write_dbt_init_completion_output( - *, request: DbtInitCommandRequest, result: DbtInitResult, use_color: bool -) -> None: - """Write dbt init preview or completion output.""" - - if request.dry_run: - print("\n" + _render_dbt_init_dry_run(result=result, use_color=use_color), end="") - return - style: CliStyle = CliStyle(use_color=use_color) - doc: CliDocument = CliDocument(style) - doc.header(text="SQLBuild project created") - doc.blank() - doc.section("Setup summary") - doc.fields( - rows=( - ("Project", result.project_name), - ("Config file", str(result.project_file)), - ("Adapter", result.adapter), - ("Target", result.target_name), - ("Profile", result.profile_name), - ), - label_width=25, - ) - doc.blank() - doc.section("What SQLBuild created") - doc.line( - f" 1. {style.value('SQLBuild twin config')} ({style.object_name('sqlbuild_project.toml')})" - ) - doc.line(" This points SQLBuild at your dbt project and profile. Edit it if dbt paths") - doc.line(" or profile targets need to change.") - doc.blank() - doc.section("Next steps") - doc.command_line(prefix=" 1. ", command=f"cd {result.output_dir}") - doc.line(" 2. " + style.warning("Review the config file above.")) - doc.command_line(prefix=" 3. ", command="sqb dbt debug") - doc.command_line(prefix=" 4. ", command="sqb dbt build") - print("\n" + doc.render(), end="") - for warning in result.warnings: - print(f"Warning: {warning}", file=sys.stderr) - - -def resolve_dbt_init_exit_code(result: DbtInitResult) -> int: - """Resolve the CLI exit code for dbt init.""" - - return 0 - - -def _render_dbt_init_dry_run(*, result: DbtInitResult, use_color: bool) -> str: - style: CliStyle = CliStyle(use_color=use_color) - doc: CliDocument = CliDocument(style) - doc.header(text="SQLBuild project preview") - doc.blank() - doc.field(label="Project", value=result.project_name) - doc.field(label="Config file", value=str(result.project_file), value_padding=" ") - doc.field(label="Adapter", value=result.adapter) - doc.field(label="Target", value=result.target_name, value_padding=" ") - doc.field(label="Profile", value=result.profile_name) - doc.blank() - doc.section("Generated config") - doc.line(result.toml.rstrip()) - return doc.render() diff --git a/src/sqlbuild/cli/commands/constants.py b/src/sqlbuild/cli/commands/constants.py index bf49cd9353..a0d40b595d 100644 --- a/src/sqlbuild/cli/commands/constants.py +++ b/src/sqlbuild/cli/commands/constants.py @@ -28,10 +28,6 @@ FAILED_STATUS: str = "failed" SCENARIO_CLI_MISSING_SUBCOMMAND: str = "C450" SQL_ANALYSIS_CONFIG_KEY: str = "sql_analysis" -DBT_JSON_OUTPUT_OPTION: str = "--json" -DBT_VERBOSE_OPTIONS: frozenset[str] = frozenset({"--verbose", "-v"}) -DBT_CLI_OUTPUT_OPTIONS: frozenset[str] = frozenset({DBT_JSON_OUTPUT_OPTION, *DBT_VERBOSE_OPTIONS}) -DBT_NO_CONNECTION_OPTION: str = "--no-connection" COST_NARROW_TERMINAL_WIDTH: int = 120 COST_HISTORY_NUMERIC_COLUMNS: frozenset[int] = frozenset({4, 5}) COST_RESOURCE_NUMERIC_START_COLUMN: int = 2 diff --git a/src/sqlbuild/cli/commands/main/dbt/_dbt.py b/src/sqlbuild/cli/commands/main/dbt/_dbt.py deleted file mode 100644 index 705c0e75a6..0000000000 --- a/src/sqlbuild/cli/commands/main/dbt/_dbt.py +++ /dev/null @@ -1,126 +0,0 @@ -"""CLI dbt interop command entry points.""" - -from __future__ import annotations - -import sys -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import PlanningProgressReporter, render_dbt_interop_plan -from sqlbuild.cli.commands._helpers.dbt.auto_init import ( - ensure_sqlbuild_project_for_dbt_command, -) -from sqlbuild.cli.commands.constants import ( - DBT_CLI_OUTPUT_OPTIONS, - DBT_JSON_OUTPUT_OPTION, - DBT_VERBOSE_OPTIONS, -) -from sqlbuild.cli.commands.main.dbt._dbt_debug import run_dbt_debug_command -from sqlbuild.integrations.dbt.main.cli.validate_execution_args import validate_dbt_execution_args -from sqlbuild.integrations.dbt.main.pipeline.execute import execute_dbt_interop_from_project -from sqlbuild.integrations.dbt.main.pipeline.plan import plan_dbt_interop_from_project -from sqlbuild.integrations.dbt.models import DbtInteropExecutionRequest, DbtInteropPlan -from sqlbuild.integrations.dbt.types import DbtInteropCommand -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.presentation.models import DisplayOptions - - -def run_dbt_command( - *, command: DbtInteropCommand, project_dir: Path | None, args: tuple[str, ...], no_color: bool -) -> int: - """Execute one `sqb dbt` interop command.""" - - validate_dbt_execution_args(command=command, args=args) - effective_project_dir: Path - forwarded_args: tuple[str, ...] - effective_project_dir, forwarded_args = ensure_sqlbuild_project_for_dbt_command( - project_dir=project_dir, - args=args, - no_color=no_color, - ) - if command == DbtInteropCommand.PLAN: - return _run_dbt_plan( - project_dir=effective_project_dir, args=forwarded_args, no_color=no_color - ) - if command == DbtInteropCommand.DEBUG: - return run_dbt_debug_command( - project_dir=effective_project_dir, args=forwarded_args, no_color=no_color - ) - return _run_dbt_execution_command( - command=command, - project_dir=effective_project_dir, - args=forwarded_args, - no_color=no_color, - ) - - -def _run_dbt_plan( - *, - project_dir: Path | None, - args: tuple[str, ...], - no_color: bool = False, -) -> int: - """Execute `sqb dbt plan`.""" - - json_output: bool = DBT_JSON_OUTPUT_OPTION in args - verbose: bool = any(option in args for option in DBT_VERBOSE_OPTIONS) - routed_args: tuple[str, ...] = tuple(arg for arg in args if arg not in DBT_CLI_OUTPUT_OPTIONS) - effective_project_dir: Path = project_dir if project_dir is not None else Path.cwd() - use_color: bool = not no_color and not json_output and supports_color() - progress_stream: TextIO = sys.stderr if json_output else sys.stdout - planning_progress: PlanningProgressReporter = PlanningProgressReporter( - stream=progress_stream, - use_color=use_color, - ) - if not json_output: - progress_stream.write("\n") - progress_stream.flush() - plan: DbtInteropPlan = plan_dbt_interop_from_project( - project_dir=effective_project_dir, - args=routed_args, - on_progress=planning_progress.on_progress, - progress_stream=progress_stream, - use_color=use_color, - ) - display_options: DisplayOptions = DisplayOptions( - max_entries_per_section=None if verbose else 10 - ) - print( - render_dbt_interop_plan( - plan=plan, - json_output=json_output, - use_color=use_color, - display_options=display_options, - ) - ) - return 0 - - -def _run_dbt_execution_command( - *, command: DbtInteropCommand, project_dir: Path | None, args: tuple[str, ...], no_color: bool -) -> int: - verbose: bool = any(option in args for option in DBT_VERBOSE_OPTIONS) - json_output: bool = DBT_JSON_OUTPUT_OPTION in args - routed_args: tuple[str, ...] = tuple(arg for arg in args if arg not in DBT_CLI_OUTPUT_OPTIONS) - effective_project_dir: Path = project_dir if project_dir is not None else Path.cwd() - use_color: bool = not no_color and not json_output and supports_color() - progress_stream: TextIO = sys.stderr if json_output else sys.stdout - planning_progress: PlanningProgressReporter = PlanningProgressReporter( - stream=progress_stream, - use_color=use_color, - ) - progress_stream.write("\n") - progress_stream.flush() - return execute_dbt_interop_from_project( - DbtInteropExecutionRequest( - command=command, - project_dir=effective_project_dir, - args=routed_args, - on_progress=planning_progress.on_progress, - progress_stream=progress_stream, - dbt_stdout_stream=sys.stdout, - use_color=use_color, - verbose=verbose, - json_output=json_output, - ) - ) diff --git a/src/sqlbuild/cli/commands/main/dbt/_dbt_debug.py b/src/sqlbuild/cli/commands/main/dbt/_dbt_debug.py deleted file mode 100644 index 7294fc02e8..0000000000 --- a/src/sqlbuild/cli/commands/main/dbt/_dbt_debug.py +++ /dev/null @@ -1,51 +0,0 @@ -"""CLI-owned dbt debug orchestration.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -from sqlbuild._native import run_debug_command -from sqlbuild.cli.commands.constants import DBT_NO_CONNECTION_OPTION -from sqlbuild.integrations.dbt.main.pipeline.debug import debug_dbt_from_project -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.presentation.main.supports_color import supports_color - - -def run_dbt_debug_command( - *, project_dir: Path | None, args: tuple[str, ...], no_color: bool -) -> int: - """Execute `sqb dbt debug`.""" - - effective_project_dir: Path = project_dir if project_dir is not None else Path.cwd() - use_color: bool = (not no_color) and supports_color() - dbt_status: TransientStatusReporter = TransientStatusReporter( - stream=sys.stderr, - use_color=use_color, - ) - dbt_status.start("Running dbt debug...") - try: - dbt_returncode: int = debug_dbt_from_project( - project_dir=effective_project_dir, - args=args, - stdout_stream=sys.stdout, - stderr_stream=sys.stderr, - ) - finally: - dbt_status.close() - sys.stdout.write("\n") - status: TransientStatusReporter = TransientStatusReporter( - stream=sys.stderr, - use_color=use_color, - ) - status.start("Running SQLBuild diagnostics...") - try: - sqlbuild_returncode: int = run_debug_command( - project_dir=project_dir, - no_color=no_color, - no_connection=DBT_NO_CONNECTION_OPTION in args, - json_output=False, - ) - finally: - status.close() - return 0 if dbt_returncode == 0 and sqlbuild_returncode == 0 else 1 diff --git a/src/sqlbuild/cli/commands/main/dbt/_dbt_init.py b/src/sqlbuild/cli/commands/main/dbt/_dbt_init.py deleted file mode 100644 index 5ef0f911ac..0000000000 --- a/src/sqlbuild/cli/commands/main/dbt/_dbt_init.py +++ /dev/null @@ -1,25 +0,0 @@ -"""CLI dbt init command entry point.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.dbt_init.execution import execute_dbt_init -from sqlbuild.cli.commands._helpers.dbt_init.invocation import resolve_dbt_init_invocation -from sqlbuild.cli.commands._helpers.dbt_init.outputs import ( - resolve_dbt_init_exit_code, - write_dbt_init_completion_output, -) -from sqlbuild.cli.commands.models import DbtInitCommandRequest, DbtInitInvocation -from sqlbuild.integrations.dbt.models import DbtInitResult - - -def run_dbt_init_command(request: DbtInitCommandRequest) -> int: - """Execute SQLBuild-owned `sqb dbt init`.""" - - invocation: DbtInitInvocation = resolve_dbt_init_invocation(request=request) - result: DbtInitResult = execute_dbt_init(invocation) - write_dbt_init_completion_output( - request=request, - result=result, - use_color=invocation.use_color, - ) - return resolve_dbt_init_exit_code(result) diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 06a5233463..30b98977e7 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -52,7 +52,6 @@ from sqlbuild.compiler.planner.models import CursorOverrides, PlanOutput from sqlbuild.compiler.refactoring.models import RefactorProject from sqlbuild.executor.janitor.models import JanitorPlan - from sqlbuild.integrations.dbt.models import DbtInitRequest @dataclass(frozen=True) @@ -273,14 +272,6 @@ class DbtInitCommandRequest: skip_dbt_debug: bool -@dataclass(frozen=True) -class DbtInitInvocation: - """Resolved profile-init request and output styling context.""" - - request: DbtInitRequest - use_color: bool - - @dataclass(frozen=True) class DiffCommandRequest: """CLI inputs for one diff command invocation.""" diff --git a/src/sqlbuild/integrations/dbt/main/cli/validate_execution_args.py b/src/sqlbuild/integrations/dbt/main/cli/_validate_execution_args.py similarity index 100% rename from src/sqlbuild/integrations/dbt/main/cli/validate_execution_args.py rename to src/sqlbuild/integrations/dbt/main/cli/_validate_execution_args.py diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py index 2005914a88..cb91d85e2b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py @@ -7,7 +7,7 @@ import pytest from _pytest.capture import CaptureResult -from sqlbuild.cli.commands.main.dbt._dbt import run_dbt_command +from sqlbuild._native import run_dbt_command from sqlbuild.integrations.dbt.models import ( DbtInitRequest, DbtInitResult, @@ -55,6 +55,7 @@ ) def test_given_dbt_plan_when_running_then_writes_progress_to_expected_stream( test_case: DbtPlanProgressTestCase, + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: @@ -73,24 +74,13 @@ def plan_dbt_interop_from_project( return build_empty_dbt_plan() monkeypatch.setattr( - "sqlbuild.cli.commands.main.dbt._dbt.plan_dbt_interop_from_project", + "sqlbuild.integrations.dbt.main.pipeline.plan.plan_dbt_interop_from_project", plan_dbt_interop_from_project, ) - - def ensure_existing_sqlbuild_project( - *, project_dir: Path | None, args: tuple[str, ...], no_color: bool - ) -> tuple[Path, tuple[str, ...]]: - del no_color - assert project_dir is not None - return project_dir, args - - monkeypatch.setattr( - "sqlbuild.cli.commands.main.dbt._dbt.ensure_sqlbuild_project_for_dbt_command", - ensure_existing_sqlbuild_project, - ) + write_minimal_sqlbuild_project(tmp_path) exit_code: int = run_dbt_command( command=DbtInteropCommand.PLAN, - project_dir=Path("/project"), + project_dir=tmp_path, args=test_case.args, no_color=True, ) @@ -125,6 +115,7 @@ def ensure_existing_sqlbuild_project( ) def test_given_dbt_execution_command_when_running_then_routes_expected_stream_and_args( test_case: DbtExecutionWrapperTestCase, + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: captured_calls: list[tuple[DbtInteropCommand, tuple[str, ...], object]] = [] @@ -134,25 +125,14 @@ def execute_dbt_interop_from_project(request: DbtInteropExecutionRequest) -> int return 0 monkeypatch.setattr( - "sqlbuild.cli.commands.main.dbt._dbt.execute_dbt_interop_from_project", + "sqlbuild.integrations.dbt.main.pipeline.execute.execute_dbt_interop_from_project", execute_dbt_interop_from_project, ) - - def ensure_existing_sqlbuild_project( - *, project_dir: Path | None, args: tuple[str, ...], no_color: bool - ) -> tuple[Path, tuple[str, ...]]: - del no_color - assert project_dir is not None - return project_dir, args - - monkeypatch.setattr( - "sqlbuild.cli.commands.main.dbt._dbt.ensure_sqlbuild_project_for_dbt_command", - ensure_existing_sqlbuild_project, - ) + write_minimal_sqlbuild_project(tmp_path) exit_code: int = run_dbt_command( command=DbtInteropCommand(test_case.command_name), - project_dir=Path("/project"), + project_dir=tmp_path, args=test_case.args, no_color=True, ) @@ -193,6 +173,7 @@ def ensure_existing_sqlbuild_project( ) def test_given_dbt_debug_command_when_running_then_invokes_dbt_and_sqlbuild_debug( test_case: DbtDebugWrapperTestCase, + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: @@ -219,37 +200,22 @@ def run_sqlbuild_debug( return 0 monkeypatch.setattr( - "sqlbuild.cli.commands.main.dbt._dbt_debug.debug_dbt_from_project", + "sqlbuild.integrations.dbt.main.pipeline.debug.debug_dbt_from_project", debug_dbt_from_project, ) - monkeypatch.setattr( - "sqlbuild.cli.commands.main.dbt._dbt_debug.run_debug_command", run_sqlbuild_debug - ) - - def ensure_existing_sqlbuild_project( - *, project_dir: Path | None, args: tuple[str, ...], no_color: bool - ) -> tuple[Path, tuple[str, ...]]: - del no_color - assert project_dir is not None - return project_dir, args - - monkeypatch.setattr( - "sqlbuild.cli.commands.main.dbt._dbt.ensure_sqlbuild_project_for_dbt_command", - ensure_existing_sqlbuild_project, - ) + monkeypatch.setattr("sqlbuild._native.run_debug_command", run_sqlbuild_debug) + write_minimal_sqlbuild_project(tmp_path) exit_code: int = run_dbt_command( command=DbtInteropCommand.DEBUG, - project_dir=Path("/project"), + project_dir=tmp_path, args=test_case.args, no_color=True, ) assert exit_code == test_case.expected_exit_code - assert dbt_calls == [(Path("/project"), test_case.expected_dbt_args, sys.stdout, sys.stderr)] - assert sqlbuild_calls == [ - (Path("/project"), True, test_case.expected_sqlbuild_no_connection, False) - ] + assert dbt_calls == [(tmp_path, test_case.expected_dbt_args, sys.stdout, sys.stderr)] + assert sqlbuild_calls == [(tmp_path, True, test_case.expected_sqlbuild_no_connection, False)] captured: CaptureResult[str] = capsys.readouterr() for fragment in test_case.expected_stderr_fragments: assert fragment in captured.err @@ -323,11 +289,11 @@ def plan_dbt_interop_from_project( return build_empty_dbt_plan() monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.dbt.auto_init.run_dbt_profile_init", + "sqlbuild.integrations.dbt.main.profile.profile_init.run_dbt_profile_init", run_dbt_profile_init, ) monkeypatch.setattr( - "sqlbuild.cli.commands.main.dbt._dbt.plan_dbt_interop_from_project", + "sqlbuild.integrations.dbt.main.pipeline.plan.plan_dbt_interop_from_project", plan_dbt_interop_from_project, ) @@ -412,11 +378,11 @@ def plan_dbt_interop_from_project( return build_empty_dbt_plan() monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.dbt.auto_init.run_dbt_profile_init", + "sqlbuild.integrations.dbt.main.profile.profile_init.run_dbt_profile_init", run_dbt_profile_init, ) monkeypatch.setattr( - "sqlbuild.cli.commands.main.dbt._dbt.plan_dbt_interop_from_project", + "sqlbuild.integrations.dbt.main.pipeline.plan.plan_dbt_interop_from_project", plan_dbt_interop_from_project, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py index 342da3075e..c67d0fc59f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py @@ -5,9 +5,9 @@ import pytest from _pytest.capture import CaptureResult -import sqlbuild.cli.commands._helpers.dbt_init.execution as dbt_init_execution -import sqlbuild.cli.commands._helpers.dbt_init.invocation as dbt_init_invocation -import sqlbuild.cli.commands.main.dbt._dbt_init as dbt_init_module +import sqlbuild._native as dbt_init_module +import sqlbuild.integrations.dbt.main.profile.profile_init as dbt_profile_init +import sqlbuild.presentation.main.supports_color as color_support from sqlbuild.cli.commands.models import DbtInitCommandRequest from sqlbuild.integrations.dbt.exceptions import DbtProfileError from sqlbuild.integrations.dbt.models import DbtInitRequest, DbtInitResult @@ -62,9 +62,9 @@ def run_dbt_profile_init(*, request: DbtInitRequest) -> DbtInitResult: toml='name = "analytics"\n', ) - monkeypatch.setattr(dbt_init_execution, "run_dbt_profile_init", run_dbt_profile_init) + monkeypatch.setattr(dbt_profile_init, "run_dbt_profile_init", run_dbt_profile_init) monkeypatch.setattr( - dbt_init_invocation, + dbt_profile_init, "_validate_dbt_profile_init_request", lambda *, request: None, ) @@ -136,9 +136,9 @@ def run_dbt_profile_init(*, request: DbtInitRequest) -> DbtInitResult: dry_run=True, ) - monkeypatch.setattr(dbt_init_execution, "run_dbt_profile_init", run_dbt_profile_init) + monkeypatch.setattr(dbt_profile_init, "run_dbt_profile_init", run_dbt_profile_init) monkeypatch.setattr( - dbt_init_invocation, + dbt_profile_init, "_validate_dbt_profile_init_request", lambda *, request: None, ) @@ -209,10 +209,10 @@ def run_dbt_profile_init(*, request: DbtInitRequest) -> DbtInitResult: toml='name = "analytics"\n', ) - monkeypatch.setattr(dbt_init_invocation, "supports_color", lambda: True) - monkeypatch.setattr(dbt_init_execution, "run_dbt_profile_init", run_dbt_profile_init) + monkeypatch.setattr(color_support, "supports_color", lambda: True) + monkeypatch.setattr(dbt_profile_init, "run_dbt_profile_init", run_dbt_profile_init) monkeypatch.setattr( - dbt_init_invocation, + dbt_profile_init, "_validate_dbt_profile_init_request", lambda *, request: None, ) @@ -272,11 +272,11 @@ def run_dbt_profile_init(*, request: DbtInitRequest) -> DbtInitResult: ) monkeypatch.setattr( - dbt_init_invocation, + dbt_profile_init, "_validate_dbt_profile_init_request", validate_dbt_profile_init_request, ) - monkeypatch.setattr(dbt_init_execution, "run_dbt_profile_init", run_dbt_profile_init) + monkeypatch.setattr(dbt_profile_init, "run_dbt_profile_init", run_dbt_profile_init) exit_code: int = dbt_init_module.run_dbt_init_command( DbtInitCommandRequest( @@ -318,11 +318,11 @@ def run_dbt_profile_init(*, request: DbtInitRequest) -> DbtInitResult: raise AssertionError("init should not run after validation failure") monkeypatch.setattr( - dbt_init_invocation, + dbt_profile_init, "_validate_dbt_profile_init_request", validate_dbt_profile_init_request, ) - monkeypatch.setattr(dbt_init_execution, "run_dbt_profile_init", run_dbt_profile_init) + monkeypatch.setattr(dbt_profile_init, "run_dbt_profile_init", run_dbt_profile_init) with pytest.raises(DbtProfileError, match=test_case.expected_error_fragment): dbt_init_module.run_dbt_init_command( From aa145e3ce594576116f02f275e05d27b72e20905 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:22:20 +0100 Subject: [PATCH 02/24] refactor(discovery): render setting guidance natively --- .../src/project/_helpers/help.rs | 11 +++ .../sqlbuild-config/src/project/constants.rs | 4 + .../project/main/explicit_references_help.rs | 25 +++++ .../src/project/main/microbatch_guidance.rs | 22 +++++ .../sqlbuild-config/src/project/main/mod.rs | 3 + .../project/main/sql_analysis_off_guidance.rs | 26 ++++++ crates/sqlbuild-config/src/project/tests.rs | 2 + .../src/project/tests/test_guidance.rs | 37 ++++++++ .../src/project/tests/test_types.rs | 7 ++ .../bindings/_helpers/discovery/guidance.rs | 60 ++++++++++++ .../src/bindings/_helpers/discovery/mod.rs | 1 + .../_helpers/discovery/registration.rs | 5 +- .../_helpers/scenario_command/setup.rs | 11 +-- src/sqlbuild/_native.pyi | 5 + .../_helpers/explicit_references/checks.py | 3 +- .../compile/_helpers/render/macros.py | 2 +- .../discovery/_helpers/settings/__init__.py | 0 .../discovery/_helpers/settings/guidance.py | 91 ------------------- .../main/_sql_analysis_off_guidance.py | 19 ---- .../main/explicit_references_help.py | 13 --- .../discovery/main/microbatch_guidance.py | 14 --- .../planner/_helpers/output/plan_entry.py | 2 +- .../setting_help/main/test_setting_help.py | 5 +- 23 files changed, 212 insertions(+), 156 deletions(-) create mode 100644 crates/sqlbuild-config/src/project/main/explicit_references_help.rs create mode 100644 crates/sqlbuild-config/src/project/main/microbatch_guidance.rs create mode 100644 crates/sqlbuild-config/src/project/main/sql_analysis_off_guidance.rs create mode 100644 crates/sqlbuild-config/src/project/tests/test_guidance.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/settings/__init__.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/settings/guidance.py delete mode 100644 src/sqlbuild/compiler/discovery/main/_sql_analysis_off_guidance.py delete mode 100644 src/sqlbuild/compiler/discovery/main/explicit_references_help.py delete mode 100644 src/sqlbuild/compiler/discovery/main/microbatch_guidance.py diff --git a/crates/sqlbuild-config/src/project/_helpers/help.rs b/crates/sqlbuild-config/src/project/_helpers/help.rs index 6b3ff98ef8..14bf4a9fe6 100644 --- a/crates/sqlbuild-config/src/project/_helpers/help.rs +++ b/crates/sqlbuild-config/src/project/_helpers/help.rs @@ -57,6 +57,17 @@ pub(crate) fn setting_note(setting: &Setting<'_>) -> String { ) } +/// `setting_note(explicit=None)`: the current value, from the file or its default. +pub(crate) fn current_value_note(setting: &Setting<'_>) -> String { + format!( + "the current value is [{}] {} = {} (from {} or its default)", + setting.section, + setting.key, + toml_value(setting.value), + setting.file_name + ) +} + /// `join_helps`: one `= help:` label per paragraph. pub(crate) fn join_helps(helps: &[String]) -> String { helps.join(ADDITIONAL_HELP_SEPARATOR) diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 3e4db042c2..c932d0a782 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -246,3 +246,7 @@ pub(crate) const GLOB_SEGMENTS: [&str; 2] = ["*", "**"]; pub(crate) const UNSUPPORTED_GLOB_MARKERS: [char; 3] = ['*', '?', '[']; /// CPython's default `sys.get_int_max_str_digits()`, beyond which `int()` raises. pub(crate) const PYTHON_INT_MAX_STR_DIGITS: usize = 4_300; +pub(crate) const MICROBATCH_CONCURRENCY_KEY: &str = "microbatch_concurrency"; +pub(crate) const REFERENCES_SECTION: &str = "references"; +pub(crate) const ENFORCE_EXPLICIT_KEY: &str = "enforce_explicit"; +pub(crate) const MICROBATCH_CONCURRENCY_PURPOSE: &str = "to run microbatches concurrently"; diff --git a/crates/sqlbuild-config/src/project/main/explicit_references_help.rs b/crates/sqlbuild-config/src/project/main/explicit_references_help.rs new file mode 100644 index 0000000000..43a8d02407 --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/explicit_references_help.rs @@ -0,0 +1,25 @@ +//! The help shown when explicit references reject an implicit reference. + +use crate::project::_helpers::help::{Setting, SettingValue, current_value_note, setting_help}; +use crate::project::constants::{ + ENFORCE_EXPLICIT_KEY, PROJECT_CONFIG_FILENAME, REFERENCES_SECTION, +}; + +/// The current-value note, then help that relaxes the policy to allow `allowed`. +#[must_use] +pub fn explicit_references_help(allowed: &str) -> String { + let setting = |value: bool| Setting { + file_name: PROJECT_CONFIG_FILENAME, + section: REFERENCES_SECTION, + key: ENFORCE_EXPLICIT_KEY, + value: SettingValue::Bool(value), + }; + format!( + "{}; {}", + current_value_note(&setting(true)), + setting_help( + &format!("while migrating a project, to allow {allowed}"), + &setting(false) + ) + ) +} diff --git a/crates/sqlbuild-config/src/project/main/microbatch_guidance.rs b/crates/sqlbuild-config/src/project/main/microbatch_guidance.rs new file mode 100644 index 0000000000..56451a16b6 --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/microbatch_guidance.rs @@ -0,0 +1,22 @@ +//! The note and help shown when microbatch concurrency is off. + +use crate::project::_helpers::help::{Setting, SettingValue, current_value_note, setting_help}; +use crate::project::constants::{ + MICROBATCH_CONCURRENCY_KEY, MICROBATCH_CONCURRENCY_PURPOSE, PROJECT_CONFIG_FILENAME, + SETTINGS_SECTION, +}; + +/// The current-value note and the help that enables microbatch concurrency. +#[must_use] +pub fn microbatch_guidance() -> (String, String) { + let setting = |value: bool| Setting { + file_name: PROJECT_CONFIG_FILENAME, + section: SETTINGS_SECTION, + key: MICROBATCH_CONCURRENCY_KEY, + value: SettingValue::Bool(value), + }; + ( + current_value_note(&setting(false)), + setting_help(MICROBATCH_CONCURRENCY_PURPOSE, &setting(true)), + ) +} diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs index 132ce189b6..e9a9a46956 100644 --- a/crates/sqlbuild-config/src/project/main/mod.rs +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -1,4 +1,7 @@ +pub mod explicit_references_help; pub mod load_local_config; pub mod load_project_config; pub mod local_sql_analysis_failure; +pub mod microbatch_guidance; pub mod resolve_config_file; +pub mod sql_analysis_off_guidance; diff --git a/crates/sqlbuild-config/src/project/main/sql_analysis_off_guidance.rs b/crates/sqlbuild-config/src/project/main/sql_analysis_off_guidance.rs new file mode 100644 index 0000000000..b5169f0cd1 --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/sql_analysis_off_guidance.rs @@ -0,0 +1,26 @@ +//! The note and help shown when SQL analysis is turned off. + +use crate::project::_helpers::help::{Setting, SettingValue, setting_help, setting_note}; +use crate::project::constants::{ + LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME, SETTINGS_SECTION, SQL_ANALYSIS_KEY, +}; + +/// The note naming the file that turns SQL analysis off and the help that turns it on for `purpose`. +#[must_use] +pub fn sql_analysis_off_guidance(local_override: bool, purpose: &str) -> (String, String) { + let file_name: &str = if local_override { + LOCAL_CONFIG_FILENAME + } else { + PROJECT_CONFIG_FILENAME + }; + let setting = |value: bool| Setting { + file_name, + section: SETTINGS_SECTION, + key: SQL_ANALYSIS_KEY, + value: SettingValue::Bool(value), + }; + ( + setting_note(&setting(false)), + setting_help(purpose, &setting(true)), + ) +} diff --git a/crates/sqlbuild-config/src/project/tests.rs b/crates/sqlbuild-config/src/project/tests.rs index 5371192a9c..6f0c538760 100644 --- a/crates/sqlbuild-config/src/project/tests.rs +++ b/crates/sqlbuild-config/src/project/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_guidance.rs"] +mod guidance; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_load_config.rs"] diff --git a/crates/sqlbuild-config/src/project/tests/test_guidance.rs b/crates/sqlbuild-config/src/project/tests/test_guidance.rs new file mode 100644 index 0000000000..b9290f28af --- /dev/null +++ b/crates/sqlbuild-config/src/project/tests/test_guidance.rs @@ -0,0 +1,37 @@ +use crate::project::main::explicit_references_help::explicit_references_help; +use crate::project::main::microbatch_guidance::microbatch_guidance; +use crate::project::main::sql_analysis_off_guidance::sql_analysis_off_guidance; +use crate::project::tests::test_types::GuidanceTestCase; + +#[test] +fn given_setting_guidance_when_rendering_then_text_matches_python() { + let test_cases = [ + GuidanceTestCase { + description: "microbatch concurrency", + rendered: microbatch_guidance(), + expected_note: "the current value is [settings] microbatch_concurrency = false (from sqlbuild_project.toml or its default)", + expected_help: "to run microbatches concurrently, set this in sqlbuild_project.toml:\n [settings]\n microbatch_concurrency = true", + }, + GuidanceTestCase { + description: "SQL analysis turned off locally", + rendered: sql_analysis_off_guidance(true, "to run X"), + expected_note: "sqlbuild_local.toml sets [settings] sql_analysis = false", + expected_help: "to run X, set this in sqlbuild_local.toml:\n [settings]\n sql_analysis = true", + }, + GuidanceTestCase { + description: "explicit references", + rendered: (explicit_references_help("X"), String::new()), + expected_note: "the current value is [references] enforce_explicit = true (from sqlbuild_project.toml or its default); while migrating a project, to allow X, set this in sqlbuild_project.toml:\n [references]\n enforce_explicit = false", + expected_help: "", + }, + ]; + + for test_case in test_cases { + assert_eq!( + (test_case.rendered.0.as_str(), test_case.rendered.1.as_str()), + (test_case.expected_note, test_case.expected_help), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/project/tests/test_types.rs b/crates/sqlbuild-config/src/project/tests/test_types.rs index bd640f62a8..a3f1dd6441 100644 --- a/crates/sqlbuild-config/src/project/tests/test_types.rs +++ b/crates/sqlbuild-config/src/project/tests/test_types.rs @@ -6,3 +6,10 @@ pub(super) struct ProjectFailureTestCase { pub(super) local: &'static str, pub(super) expected_failure: Option, } + +pub(super) struct GuidanceTestCase { + pub(super) description: &'static str, + pub(super) rendered: (String, String), + pub(super) expected_note: &'static str, + pub(super) expected_help: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs new file mode 100644 index 0000000000..7c94d567db --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs @@ -0,0 +1,60 @@ +//! Setting guidance shown by diagnostics that depend on project settings. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_config::project::main::explicit_references_help::explicit_references_help as references_help; +use sqlbuild_config::project::main::microbatch_guidance::microbatch_guidance as concurrency_guidance; +use sqlbuild_config::project::main::sql_analysis_off_guidance::sql_analysis_off_guidance as analysis_off; + +const SQL_ANALYSIS_KEY: &str = "sql_analysis"; + +/// The note and help for turning SQL analysis on, or `None` when it is effectively on. +pub(crate) fn sql_analysis_guidance( + discovered_inputs: &Bound<'_, PyAny>, + purpose: &str, +) -> PyResult> { + let local: Bound<'_, PyAny> = discovered_inputs.getattr("local_config")?; + let local_override: bool = local + .getattr("setting_overrides")? + .contains(SQL_ANALYSIS_KEY)?; + let owner: Bound<'_, PyAny> = if local_override { + local + } else { + discovered_inputs.getattr("project_config")? + }; + if owner + .getattr("settings")? + .getattr(SQL_ANALYSIS_KEY)? + .is_truthy()? + { + return Ok(None); + } + Ok(Some(analysis_off(local_override, purpose))) +} + +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, purpose))] +fn sql_analysis_off_guidance( + discovered_inputs: &Bound<'_, PyAny>, + purpose: &str, +) -> PyResult> { + sql_analysis_guidance(discovered_inputs, purpose) +} + +#[pyfunction] +fn microbatch_guidance() -> (String, String) { + concurrency_guidance() +} + +#[pyfunction] +#[pyo3(signature = (*, allowed))] +fn explicit_references_help(allowed: &str) -> String { + references_help(allowed) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(sql_analysis_off_guidance, module)?)?; + module.add_function(wrap_pyfunction!(microbatch_guidance, module)?)?; + module.add_function(wrap_pyfunction!(explicit_references_help, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index 196bc9c635..9b62cc9132 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod declaration_files; pub(crate) mod declaration_records; pub(crate) mod faults; +pub(crate) mod guidance; pub(crate) mod header_declarations; pub(crate) mod header_values; pub(crate) mod key_checks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs index aceed8af24..9bcc725c6d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs @@ -3,8 +3,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::discovery::{ - declaration_files, declaration_records, header_declarations, header_values, key_checks, - locations, model_records, project_files, project_validation, python_function_files, + declaration_files, declaration_records, guidance, header_declarations, header_values, + key_checks, locations, model_records, project_files, project_validation, python_function_files, python_hook_files, python_module_files, raised, scope_snapshot, snapshot, sql_test_records, yaml_entries, yaml_records, }; @@ -28,5 +28,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { project_validation::register(module)?; yaml_entries::register(module)?; header_values::register(module)?; + guidance::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index 512834423f..f4fddaa6b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -21,11 +21,11 @@ use sqlbuild_core::scenario_command::models::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::discovery::guidance::sql_analysis_guidance; pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const GUIDANCE_MODULE: &str = "sqlbuild.compiler.discovery.main._sql_analysis_off_guidance"; const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; const SCENARIO_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_scenario_config"; @@ -180,14 +180,7 @@ pub(crate) fn require_sql_analysis( let guidance: Option<(String, String)> = if disabled_by_flag { None } else { - call_with( - &imported(py, GUIDANCE_MODULE, "sql_analysis_off_guidance")?, - &[ - ("discovered_inputs", project.discovered.clone()), - ("purpose", text(py, sql_analysis_purpose(command))), - ], - )? - .extract()? + sql_analysis_guidance(&project.discovered, sql_analysis_purpose(command))? }; match sql_analysis_required_error(command, disabled_by_flag, guidance) { Some(error) => Err(cli_user_error(py, error)), diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index e7fa81c4b1..05fa724c6f 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2966,6 +2966,11 @@ def run_dag_command( ) -> int: ... def run_scenario_test_command(*, request: ScenarioTestCommandRequest) -> int: ... def run_scenario_capture_command(*, request: ScenarioCaptureCommandRequest) -> int: ... +def sql_analysis_off_guidance( + *, discovered_inputs: DiscoveredProjectInputs, purpose: str +) -> tuple[str, str] | None: ... +def microbatch_guidance() -> tuple[str, str]: ... +def explicit_references_help(*, allowed: str) -> str: ... def run_format_command( *, project_dir: Path | None, diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py index 7ea879a863..6859c703c8 100644 --- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py +++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py @@ -9,10 +9,9 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import model_python_hook_names +from sqlbuild._native import explicit_references_help, model_python_hook_names from sqlbuild.compiler.compile.models import PythonSqlReferenceReport from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.main.explicit_references_help import explicit_references_help from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py index 4febf59601..649e7edd91 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py @@ -16,6 +16,7 @@ from typing import cast import sqlbuild._native as _native +from sqlbuild._native import explicit_references_help from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( call_site_sql_references, @@ -42,7 +43,6 @@ StaticMacroFault, StaticMacroInventory, ) -from sqlbuild.compiler.discovery.main.explicit_references_help import explicit_references_help from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, DiscoveredMacroFile, diff --git a/src/sqlbuild/compiler/discovery/_helpers/settings/__init__.py b/src/sqlbuild/compiler/discovery/_helpers/settings/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/discovery/_helpers/settings/guidance.py b/src/sqlbuild/compiler/discovery/_helpers/settings/guidance.py deleted file mode 100644 index 8a7f24facf..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/settings/guidance.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Current values of project settings and the exact TOML that changes them.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery.constants import ( - ENFORCE_EXPLICIT_REFERENCES_KEY, - LOCAL_CONFIG_FILENAME, - MICROBATCH_CONCURRENCY_SETTING_KEY, - PROJECT_CONFIG_FILENAME, - REFERENCES_SECTION, - SETTINGS_SECTION, - SQL_ANALYSIS_CONFIG_KEY, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.errors.setting_help.main.setting_help import setting_help -from sqlbuild.errors.setting_help.main.setting_note import setting_note - - -def effective_sql_analysis(discovered_inputs: DiscoveredProjectInputs) -> bool: - """Return the project SQL analysis setting after local overrides.""" - - if SQL_ANALYSIS_CONFIG_KEY in discovered_inputs.local_config.setting_overrides: - return discovered_inputs.local_config.settings.sql_analysis - return discovered_inputs.project_config.settings.sql_analysis - - -def sql_analysis_off_note_and_help( - *, discovered_inputs: DiscoveredProjectInputs, purpose: str -) -> tuple[str, str]: - """Return a note naming the file that turns SQL analysis off and help to turn it on.""" - - file_name: str = ( - LOCAL_CONFIG_FILENAME - if SQL_ANALYSIS_CONFIG_KEY in discovered_inputs.local_config.setting_overrides - else PROJECT_CONFIG_FILENAME - ) - return ( - setting_note( - file_name=file_name, section=SETTINGS_SECTION, key=SQL_ANALYSIS_CONFIG_KEY, value=False - ), - setting_help( - purpose=purpose, - file_name=file_name, - section=SETTINGS_SECTION, - key=SQL_ANALYSIS_CONFIG_KEY, - value=True, - ), - ) - - -def microbatch_concurrency_note() -> str: - """State that concurrent microbatches are off in the project file.""" - - return setting_note( - file_name=PROJECT_CONFIG_FILENAME, - section=SETTINGS_SECTION, - key=MICROBATCH_CONCURRENCY_SETTING_KEY, - value=False, - explicit=None, - ) - - -def microbatch_concurrency_help() -> str: - """Show the exact setting that allows `batch_concurrency` above one.""" - - return setting_help( - purpose="to run microbatches concurrently", - file_name=PROJECT_CONFIG_FILENAME, - section=SETTINGS_SECTION, - key=MICROBATCH_CONCURRENCY_SETTING_KEY, - value=True, - ) - - -def explicit_references_help(*, allowed: str) -> str: - """Show the exact migration setting that relaxes explicit references, with its value.""" - - note: str = setting_note( - file_name=PROJECT_CONFIG_FILENAME, - section=REFERENCES_SECTION, - key=ENFORCE_EXPLICIT_REFERENCES_KEY, - value=True, - explicit=None, - ) - return f"{note}; " + setting_help( - purpose=f"while migrating a project, to allow {allowed}", - file_name=PROJECT_CONFIG_FILENAME, - section=REFERENCES_SECTION, - key=ENFORCE_EXPLICIT_REFERENCES_KEY, - value=False, - ) diff --git a/src/sqlbuild/compiler/discovery/main/_sql_analysis_off_guidance.py b/src/sqlbuild/compiler/discovery/main/_sql_analysis_off_guidance.py deleted file mode 100644 index 694fb5b000..0000000000 --- a/src/sqlbuild/compiler/discovery/main/_sql_analysis_off_guidance.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Entry for the note and help shown when SQL analysis is turned off by a config file.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery._helpers.settings.guidance import ( - effective_sql_analysis, - sql_analysis_off_note_and_help, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs - - -def sql_analysis_off_guidance( - *, discovered_inputs: DiscoveredProjectInputs, purpose: str -) -> tuple[str, str] | None: - """Return `(note, help)` naming the file that turns SQL analysis off, or None when it is on.""" - - if effective_sql_analysis(discovered_inputs): - return None - return sql_analysis_off_note_and_help(discovered_inputs=discovered_inputs, purpose=purpose) diff --git a/src/sqlbuild/compiler/discovery/main/explicit_references_help.py b/src/sqlbuild/compiler/discovery/main/explicit_references_help.py deleted file mode 100644 index 4f2299973a..0000000000 --- a/src/sqlbuild/compiler/discovery/main/explicit_references_help.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Entry for the migration help that relaxes explicit references.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery._helpers.settings.guidance import ( - explicit_references_help as _explicit_references_help, -) - - -def explicit_references_help(*, allowed: str) -> str: - """Show the current `[references] enforce_explicit` value and the exact TOML to relax it.""" - - return _explicit_references_help(allowed=allowed) diff --git a/src/sqlbuild/compiler/discovery/main/microbatch_guidance.py b/src/sqlbuild/compiler/discovery/main/microbatch_guidance.py deleted file mode 100644 index 4980bbcccd..0000000000 --- a/src/sqlbuild/compiler/discovery/main/microbatch_guidance.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Entry for the note and help shown when concurrent microbatches are not allowed.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery._helpers.settings.guidance import ( - microbatch_concurrency_help, - microbatch_concurrency_note, -) - - -def microbatch_guidance() -> tuple[str, str]: - """Return `(note, help)` for `batch_concurrency` above one without the project capability.""" - - return microbatch_concurrency_note(), microbatch_concurrency_help() diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index b33e29bad2..887ee0c851 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -16,6 +16,7 @@ get_config_int, get_config_str, get_config_string_tuple, + microbatch_guidance, render_source_relation, resolve_cursor_input_roles, ) @@ -40,7 +41,6 @@ CompiledResourceType, CompiledSeedView, ) -from sqlbuild.compiler.discovery.main.microbatch_guidance import microbatch_guidance from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.graph.source_load_nodes import build_source_load_map from sqlbuild.compiler.planner._helpers.output.cursor_type_check import ( diff --git a/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py b/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py index 104d32d16c..4f1ef80581 100644 --- a/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py +++ b/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py @@ -6,11 +6,8 @@ from scripts.setting_help.main.find_settings_without_snippet import find_settings_without_snippet from scripts.setting_help.main.parse_setting_snippet import parse_setting_snippet +from sqlbuild._native import explicit_references_help, microbatch_guidance from sqlbuild.compiler.compile._helpers.analysis.validation import sql_analysis_opt_out_help -from sqlbuild.compiler.discovery.main.explicit_references_help import explicit_references_help -from sqlbuild.compiler.discovery.main.microbatch_guidance import ( - microbatch_guidance, -) from sqlbuild.errors.setting_help.main.join_helps import join_helps from sqlbuild.errors.setting_help.main.model_header_help import model_header_help from sqlbuild.errors.setting_help.main.setting_help import setting_help From 85cd9b0f567ba55b86d9793d33816dd88bac58bb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:28:42 +0100 Subject: [PATCH 03/24] refactor(discovery): build configuration-only and selected-contract inputs natively --- .../_helpers/command_runtime/warehouses.rs | 12 +-- .../_helpers/compile_driver/discovery.rs | 12 ++- .../bindings/_helpers/diff_report/command.rs | 7 +- .../_helpers/discovery/configuration.rs | 44 ++++++++++ .../src/bindings/_helpers/discovery/mod.rs | 1 + .../_helpers/format_command/selection.rs | 87 ++++++++++++++----- .../_helpers/integrations/loaders.py | 16 ---- .../_helpers/validation/supported_keys.py | 26 ------ .../compiler/discovery/classes/__init__.py | 0 .../selected_contract_input_discoverer.py | 55 ------------ .../discovery/main/_discover_configuration.py | 20 ----- .../main/_model_output_column_locations.py | 23 ----- .../main/omitted_ceremonial_select.py | 12 --- .../sqlbuild/compiler/discovery/helpers.py | 23 +++-- .../discovery/_helpers/test_sql_tests.py | 6 +- 15 files changed, 141 insertions(+), 203 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/validation/supported_keys.py delete mode 100644 src/sqlbuild/compiler/discovery/classes/__init__.py delete mode 100644 src/sqlbuild/compiler/discovery/classes/selected_contract_input_discoverer.py delete mode 100644 src/sqlbuild/compiler/discovery/main/_discover_configuration.py delete mode 100644 src/sqlbuild/compiler/discovery/main/_model_output_column_locations.py delete mode 100644 src/sqlbuild/compiler/discovery/main/omitted_ceremonial_select.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index be33723125..0ac831302d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -22,14 +22,13 @@ use crate::bindings::_helpers::command_runtime::runtime_errors::{ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::discovery::configuration::discover_project_configuration; const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; const SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main.session_warehouse_support"; -const DISCOVER_CONFIGURATION_MODULE: &str = - "sqlbuild.compiler.discovery.main._discover_configuration"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; const EXPAND_TEMPLATE_MODULE: &str = "sqlbuild.compiler.compile.main.expand_template_data"; const RESOLVE_OPTIONS: (&[&str], &[&str]) = ( @@ -304,14 +303,7 @@ fn validate_cli_warehouse( return Ok(()); }; checked_cli_warehouse(py, &cli_warehouse)?; - let discovered: Bound<'_, PyAny> = call_with( - &imported( - py, - DISCOVER_CONFIGURATION_MODULE, - "discover_project_configuration", - )?, - &[("project_dir", project_dir.clone())], - )?; + let discovered: Bound<'_, PyAny> = discover_project_configuration(py, &project_dir)?; let adapter_name: Bound<'_, PyAny> = call_with( &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index f8c7aba53c..325e0605de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -37,7 +37,8 @@ const EVENT_EXPORTERS_MODULE: &str = "sqlbuild.runtime.event_exporting.main._configure_discovered_event_exporters"; const FILESYSTEM_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.core"; const SINKS_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks"; -const LOADERS_MODULE: &str = "sqlbuild.compiler.discovery._helpers.integrations.loaders"; +const INGESTR_LOADERS_MODULE: &str = "sqlbuild.integrations.ingestr.main.loaders"; +const DLT_LOADERS_MODULE: &str = "sqlbuild.integrations.dlt.main.loaders"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; /// Declaration file collections counted into the declaration-parse lifecycle, in field order. @@ -455,9 +456,14 @@ fn project_inputs<'py>( )? .into_any()) }; + let source_files: Bound<'py, PyAny> = declarations.get_item("source_files")?; let integration_loaders: Bound<'py, PyAny> = - imported(py, LOADERS_MODULE, "build_integration_loader_functions")? - .call1((declarations.get_item("source_files")?,))?; + imported(py, INGESTR_LOADERS_MODULE, "build_ingestr_loader_functions")? + .call1((&source_files,))? + .add( + imported(py, DLT_LOADERS_MODULE, "build_dlt_loader_functions")? + .call1((&source_files,))?, + )?; let loader_functions: Bound<'py, PyAny> = tuple_of("loaders")?.add(integration_loaders)?; let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ ("project_config", project_config.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs index 164f2b106b..bd17fa0151 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -26,10 +26,10 @@ use crate::bindings::_helpers::diff_report::support::{ EXCEPTIONS_MODULE, call, caused, cli_error, error_text, is_exception, matches_class, none, object, request_facts, user_error_parts, }; +use crate::bindings::_helpers::discovery::configuration::discover_project_configuration; use crate::bindings::_helpers::frontier::encoder::emit_error; const NATIVE_MODULE: &str = "sqlbuild._native"; -const CONFIGURATION_MODULE: &str = "sqlbuild.compiler.discovery.main._discover_configuration"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; @@ -85,10 +85,7 @@ fn resolve_invocation<'py>( _ => imported(py, "pathlib", "Path")?.call_method0("cwd")?, }; let discovered: Bound<'py, PyAny> = if query_mode { - call( - &imported(py, CONFIGURATION_MODULE, "discover_project_configuration")?, - &[("project_dir", &project_dir)], - )? + discover_project_configuration(py, &project_dir)? } else { call( &imported(py, NATIVE_MODULE, "discover_project_inputs")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs new file mode 100644 index 0000000000..939191d8de --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs @@ -0,0 +1,44 @@ +//! Project inputs holding only the project and local configuration. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; + +/// `DiscoveredProjectInputs` with the loaded configuration, inside the project discovery lifecycle. +pub(crate) fn discover_project_configuration<'py>( + py: Python<'py>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let lifecycle: Bound<'py, PyAny> = call_with( + &imported(py, LIFECYCLE_MODULE, "OperationLifecycle")?, + &[ + ("operation_kind", PyString::new(py, "project").into_any()), + ( + "operation_name", + PyString::new(py, "project_discovery").into_any(), + ), + ], + )?; + within(&lifecycle, |_| { + call_with( + &imported(py, DISCOVERY_MODELS_MODULE, "DiscoveredProjectInputs")?, + &[ + ("project_dir", project_dir.clone()), + ( + "project_config", + imported(py, NATIVE_MODULE, "load_project_config_file")? + .call1((project_dir,))?, + ), + ( + "local_config", + imported(py, NATIVE_MODULE, "load_local_config_file")?.call1((project_dir,))?, + ), + ], + ) + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index 9b62cc9132..f70928f6fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod configuration; pub(crate) mod declaration_files; pub(crate) mod declaration_records; pub(crate) mod faults; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index 8dfbcbbaf9..1508e8bbdf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -14,12 +14,11 @@ use sqlbuild_core::format_command::main::resolve_lint_paths::resolve_lint_paths; use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_null_candidates; use crate::bindings::_helpers::boundary::panics::value_error; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; const PLANNER_ERRORS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; -const SELECTED_DISCOVERER_MODULE: &str = - "sqlbuild.compiler.discovery.classes.selected_contract_input_discoverer"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const UNIT_TEST_SELECTORS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; @@ -240,6 +239,66 @@ fn combined_paths( Ok(Some(selected.difference(&excluded).cloned().collect())) } +/// The contract inputs selected fixtures need, without importing Python runtime resources. +fn selected_contract_inputs<'py>( + project_dir: &Bound<'py, PyAny>, + selected_test_paths: Bound<'py, PyAny>, + referenced_model_names: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let native = |name: &str| imported(py, NATIVE_MODULE, name); + let project_config: Bound<'py, PyAny> = + native("load_project_config_file")?.call1((project_dir,))?; + let local_config: Bound<'py, PyAny> = + native("load_local_config_file")?.call1((project_dir,))?; + let disabled: Bound<'py, PyAny> = PyBool::new(py, false).to_owned().into_any(); + let scope: Bound<'py, PyAny> = call_with( + &native("directory_snapshot_scope")?, + &[("project_dir", project_dir.clone())], + )?; + within(&scope, |_| { + let project_only = |name: &str| -> PyResult> { + call_with(&native(name)?, &[("project_dir", project_dir.clone())]) + }; + call_with( + &imported(py, DISCOVERY_MODELS_MODULE, "DiscoveredProjectInputs")?, + &[ + ("project_config", project_config), + ("local_config", local_config), + ("project_dir", project_dir.clone()), + ( + "model_files", + call_with( + &native("discover_model_files")?, + &[ + ("project_dir", project_dir.clone()), + ("extract_implicit_alias_columns", disabled.clone()), + ("extract_output_column_locations", disabled.clone()), + ("selected_model_names", referenced_model_names), + ], + )?, + ), + ( + "model_schema_files", + project_only("discover_model_schema_files")?, + ), + ("schema_files", project_only("discover_schema_files")?), + ("source_files", project_only("discover_source_files")?), + ( + "test_files", + call_with( + &native("discover_test_files")?, + &[ + ("project_dir", project_dir.clone()), + ("selected_paths", selected_test_paths), + ], + )?, + ), + ], + ) + }) +} + /// Inputs for a run scoped by path selectors, discovering only the contracts fixtures need. fn path_scoped_inputs<'py>( project_dir: &Bound<'py, PyAny>, @@ -250,24 +309,10 @@ fn path_scoped_inputs<'py>( validate(py, &paths)?; let candidates = scan_fixture_null_candidates(directory, Some(&paths)).map_err(value_error)?; let candidate_paths: Vec = candidates.paths.into_iter().map(PathBuf::from).collect(); - let discovered: Bound<'py, PyAny> = call_with( - &imported( - py, - SELECTED_DISCOVERER_MODULE, - "SelectedContractInputDiscoverer", - )? - .getattr("discover")?, - &[ - ("project_dir", project_dir.clone()), - ( - "selected_test_paths", - PyFrozenSet::new(py, candidate_paths)?.into_any(), - ), - ( - "referenced_model_names", - PyFrozenSet::new(py, candidates.model_names)?.into_any(), - ), - ], + let discovered: Bound<'py, PyAny> = selected_contract_inputs( + project_dir, + PyFrozenSet::new(py, candidate_paths)?.into_any(), + PyFrozenSet::new(py, candidates.model_names)?.into_any(), )?; Ok(LintInputs { adapter: discovered_adapter(project_dir, &discovered)?, diff --git a/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py b/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py deleted file mode 100644 index e5612a27ce..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Discovery helpers for declarative integration loaders.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction, DiscoveredSourceFile - - -def build_integration_loader_functions( - source_files: tuple[DiscoveredSourceFile, ...], -) -> tuple[DiscoveredLoaderFunction, ...]: - """Build synthetic loader functions for declarative integration loaders.""" - - from sqlbuild.integrations.dlt.main.loaders import build_dlt_loader_functions - from sqlbuild.integrations.ingestr.main.loaders import build_ingestr_loader_functions - - return build_ingestr_loader_functions(source_files) + build_dlt_loader_functions(source_files) diff --git a/src/sqlbuild/compiler/discovery/_helpers/validation/supported_keys.py b/src/sqlbuild/compiler/discovery/_helpers/validation/supported_keys.py deleted file mode 100644 index a9e0d30386..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/validation/supported_keys.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Shared rejection of unsupported keys in authored configuration mappings.""" - -from __future__ import annotations - -from collections.abc import Mapping -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery.exceptions import DiscoveryError - - -def reject_unknown_mapping_keys( - *, - mapping: Mapping[str, object], - allowed: frozenset[str], - file_path: Path, - label: str, - error_class: type[DiscoveryError], -) -> None: - """Raise when a YAML mapping declares keys outside its allowed set.""" - - failure: tuple[str, str] | None = _native.unknown_mapping_keys( - [(str(key), isinstance(key, str)) for key in mapping], list(allowed) - ) - if failure is not None: - raise error_class(f"{file_path} {label} {failure[0]}", help=failure[1]) diff --git a/src/sqlbuild/compiler/discovery/classes/__init__.py b/src/sqlbuild/compiler/discovery/classes/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/discovery/classes/selected_contract_input_discoverer.py b/src/sqlbuild/compiler/discovery/classes/selected_contract_input_discoverer.py deleted file mode 100644 index cbc969d526..0000000000 --- a/src/sqlbuild/compiler/discovery/classes/selected_contract_input_discoverer.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Bounded discovery for selected contract consumers.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild._native import ( - directory_snapshot_scope, - discover_model_files, - discover_model_schema_files, - discover_schema_files, - discover_source_files, - discover_test_files, -) -from sqlbuild.compiler.discovery._helpers.yml.project import ( - load_local_config, - load_project_config, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -class SelectedContractInputDiscoverer: - """Discover contracts and selected SQL tests without unrelated runtime resources.""" - - @staticmethod - def discover( - *, - project_dir: Path, - selected_test_paths: frozenset[Path], - referenced_model_names: frozenset[str], - ) -> DiscoveredProjectInputs: - """Build selected contract inputs without importing Python runtime resources.""" - - project_config: ProjectConfig = load_project_config(project_dir=project_dir) - local_config: LocalConfig = load_local_config(project_dir=project_dir) - with directory_snapshot_scope(project_dir=project_dir): - return DiscoveredProjectInputs( - project_config=project_config, - local_config=local_config, - project_dir=project_dir, - model_files=discover_model_files( - project_dir=project_dir, - extract_implicit_alias_columns=False, - extract_output_column_locations=False, - selected_model_names=referenced_model_names, - ), - model_schema_files=discover_model_schema_files(project_dir=project_dir), - schema_files=discover_schema_files(project_dir=project_dir), - source_files=discover_source_files(project_dir=project_dir), - test_files=discover_test_files( - project_dir=project_dir, - selected_paths=selected_test_paths, - ), - ) diff --git a/src/sqlbuild/compiler/discovery/main/_discover_configuration.py b/src/sqlbuild/compiler/discovery/main/_discover_configuration.py deleted file mode 100644 index 5dbb1abac8..0000000000 --- a/src/sqlbuild/compiler/discovery/main/_discover_configuration.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Lightweight project configuration discovery entrypoint.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle - - -def discover_project_configuration(*, project_dir: Path) -> DiscoveredProjectInputs: - """Load project and local configuration without importing project extensions.""" - - with OperationLifecycle(operation_kind="project", operation_name="project_discovery"): - return DiscoveredProjectInputs( - project_dir=project_dir, - project_config=load_project_config(project_dir=project_dir), - local_config=load_local_config(project_dir=project_dir), - ) diff --git a/src/sqlbuild/compiler/discovery/main/_model_output_column_locations.py b/src/sqlbuild/compiler/discovery/main/_model_output_column_locations.py deleted file mode 100644 index d07007a8b7..0000000000 --- a/src/sqlbuild/compiler/discovery/main/_model_output_column_locations.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Public access to authored model output-column locations.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild._native import model_output_column_locations -from sqlbuild.spec.contracts.models import SourceLocation - - -def extract_model_output_column_locations( - *, - contents: str, - relative_path: Path, - extract_implicit_alias_columns: bool, -) -> dict[str, SourceLocation]: - """Locate authored output projections only when a compiler consumer needs them.""" - - return model_output_column_locations( - contents=contents, - relative_path=relative_path, - extract_implicit_alias_columns=extract_implicit_alias_columns, - ) diff --git a/src/sqlbuild/compiler/discovery/main/omitted_ceremonial_select.py b/src/sqlbuild/compiler/discovery/main/omitted_ceremonial_select.py deleted file mode 100644 index 6de00942c7..0000000000 --- a/src/sqlbuild/compiler/discovery/main/omitted_ceremonial_select.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Locate where a SQL test or scenario body omits its trailing ceremonial `SELECT 1`.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -def omitted_ceremonial_select_offset(*, sql: str, syntax: SqlLexicalSyntax) -> int | None: - """Return the offset after a body's last top-level CTE when it ends without `SELECT 1`.""" - - return _native.omitted_ceremonial_select(sql, syntax.native_mapping) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index 4b89ea9e54..e272e693ff 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -31,9 +31,6 @@ discover_sql_hook_files, render_stage_capture, ) -from sqlbuild.compiler.discovery.classes.selected_contract_input_discoverer import ( - SelectedContractInputDiscoverer, -) from sqlbuild.compiler.discovery.constants import ( SQL_AUDIT_HEADER_KEYS, SQL_FUNCTION_HEADER_KEYS, @@ -44,6 +41,7 @@ ) from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, + DiscoveredSourceFile, DiscoveredSqlModelFile, DiscoveryFileFault, NamedSqlHookEntry, @@ -536,12 +534,21 @@ def description_inputs_outcome(*, project_dir: Path) -> tuple[object, ...]: def selected_contract_outcome(*, project_dir: Path) -> tuple[object, ...]: """Return the model and source paths selected-contract discovery reads for `orders`.""" - inputs: DiscoveredProjectInputs = SelectedContractInputDiscoverer.discover( - project_dir=project_dir, - selected_test_paths=frozenset(), - referenced_model_names=frozenset({"orders"}), + with _native.directory_snapshot_scope(project_dir=project_dir): + models: tuple[DiscoveredSqlModelFile, ...] = _native.discover_model_files( + project_dir=project_dir, + extract_implicit_alias_columns=False, + extract_output_column_locations=False, + selected_model_names=frozenset({"orders"}), + ) + sources: tuple[DiscoveredSourceFile, ...] = _native.discover_source_files( + project_dir=project_dir + ) + return ( + tuple(model.relative_path.as_posix() for model in models), + tuple(source.relative_path.as_posix() for source in sources), + (), ) - return (*_inputs_paths(inputs), ()) def _inputs_paths(inputs: DiscoveredProjectInputs) -> tuple[object, ...]: diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py index 3f28cc8e09..30facdede3 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py @@ -8,11 +8,9 @@ from sqlbuild._native import ( discover_test_files, + omitted_ceremonial_select, ) from sqlbuild.compiler.discovery.exceptions import SqlTestParseError -from sqlbuild.compiler.discovery.main.omitted_ceremonial_select import ( - omitted_ceremonial_select_offset, -) from sqlbuild.compiler.discovery.models import ( DiscoveredSqlTestBlock, DiscoveredSqlTestCase, @@ -76,7 +74,7 @@ def test_given_large_fixture_body_when_locating_omitted_select_then_completes_wi for _ in range(3): started: float = time.perf_counter() - offset = omitted_ceremonial_select_offset(sql=sql, syntax=SqlLexicalSyntax()) + offset = omitted_ceremonial_select(sql, SqlLexicalSyntax().native_mapping) timings.append(time.perf_counter() - started) assert (offset is not None) == test_case.expected_offset_found From f84344b936850e8bfe4e491270744bcac6a50ef1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:10:26 +0100 Subject: [PATCH 04/24] chore: keep discovery modules that import project Python as user-code --- python-boundary.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/python-boundary.toml b/python-boundary.toml index 5ed091ad03..290d4509c5 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -179,6 +179,21 @@ path = "compiler/python_nodes/_helpers/identity.py" reason = "user-code" note = "Identity hashing over live user functions (source and globals)." +[[stays]] +path = "compiler/discovery/_helpers/filesystem/core.py" +reason = "user-code" +note = "Imports project Python node, provider, sink and materialization modules and reads their definitions." + +[[stays]] +path = "compiler/discovery/_helpers/validation/discovery.py" +reason = "user-code" +note = "Reads decorator definitions from loaded project Python node functions." + +[[stays]] +path = "compiler/discovery/main/runtime_extensions.py" +reason = "user-code" +note = "Binds imported project providers to event exporter and sink declarations." + [[stays]] path = "compiler/discovery/_helpers/filesystem/command_output_sinks.py" reason = "user-code" From 3022198aace672dad2d00329301030fbaa61df8c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:15:03 +0100 Subject: [PATCH 05/24] refactor(discovery): serialize model hook entries natively --- .../_helpers/compile_command/report.rs | 16 ++-- .../_helpers/discovery/hook_entries.rs | 86 +++++++++++++++++++ .../src/bindings/_helpers/discovery/mod.rs | 1 + .../_helpers/discovery/registration.rs | 7 +- src/sqlbuild/_native.pyi | 6 ++ .../discovery/_helpers/sql/hook_entries.py | 53 ------------ .../discovery/main/serialize_hook_entries.py | 20 ----- .../version_identity_model_metadata.py | 2 +- .../main/test_serialize_hook_entries.py | 2 +- 9 files changed, 105 insertions(+), 88 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/hook_entries.rs delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/sql/hook_entries.py delete mode 100644 src/sqlbuild/compiler/discovery/main/serialize_hook_entries.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index 4865d9f632..c7b0e058e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -18,9 +18,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::compile_outputs::json_report::{ emit_json_report, emit_orjson_report, }; +use crate::bindings::_helpers::discovery::hook_entries::hook_entries; const SEMANTIC_COVERAGE_MODULE: &str = "sqlbuild.compiler.compile.main._semantic_coverage"; -const HOOK_ENTRIES_MODULE: &str = "sqlbuild.compiler.discovery.main.serialize_hook_entries"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; @@ -634,15 +634,11 @@ fn model_resource<'py>( if !hooks.is_none() { item.set_item( hook_key, - call_with( - &imported(py, HOOK_ENTRIES_MODULE, "serialize_hook_entries")?, - &[ - ("value", hooks), - ( - "sql_fields", - imported(py, DISCOVERY_CONSTANTS_MODULE, "SQL_HOOK_OUTPUT_FIELDS")?, - ), - ], + hook_entries( + &hooks, + &imported(py, DISCOVERY_CONSTANTS_MODULE, "SQL_HOOK_OUTPUT_FIELDS")? + .extract::>()?, + None, )?, )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/hook_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/hook_entries.rs new file mode 100644 index 0000000000..0b3efe5ed7 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/hook_entries.rs @@ -0,0 +1,86 @@ +//! Model lifecycle hooks as the plain dictionaries manifests and identities record. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const RELATIVE_PATH_FIELD: &str = "relative_path"; +const SQL_HOOK_FIELDS: [&str; 6] = [ + "statement", + "name", + "relative_path", + "definition_sql", + "kwargs", + "description", +]; + +fn sql_hook<'py>(entry: &Bound<'py, PyAny>, sql_fields: &[String]) -> PyResult> { + let hook: Bound<'py, PyDict> = PyDict::new(entry.py()); + hook.set_item("type", "sql")?; + for field in sql_fields { + let Some(name) = SQL_HOOK_FIELDS.iter().find(|name| **name == field) else { + return Err(pyo3::exceptions::PyKeyError::new_err(field.clone())); + }; + let mut value: Bound<'py, PyAny> = entry.getattr(*name)?; + if value.is_none() { + continue; + } + if *name == RELATIVE_PATH_FIELD { + value = value.call_method0("as_posix")?; + } + hook.set_item(*name, value)?; + } + Ok(hook) +} + +/// Serialize SQL and Python hook entries; other values serialize to no hooks. +pub(crate) fn hook_entries<'py>( + value: &Bound<'py, PyAny>, + sql_fields: &[String], + python_hook_fields: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let py = value.py(); + let hooks: Bound<'py, PyList> = PyList::empty(py); + if !value.is_instance_of::() && !value.is_instance_of::() { + return Ok(hooks); + } + let sql_class: Bound<'py, PyAny> = imported(py, DISCOVERY_MODELS_MODULE, "SqlHookEntry")?; + let python_class: Bound<'py, PyAny> = imported(py, DISCOVERY_MODELS_MODULE, "PythonHookEntry")?; + for entry in value.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + if entry.is_instance(&sql_class)? { + hooks.append(sql_hook(&entry, sql_fields)?)?; + } else if entry.is_instance(&python_class)? { + let hook: Bound<'py, PyDict> = PyDict::new(py); + let name: Bound<'py, PyAny> = entry.getattr("name")?; + hook.set_item("type", "python")?; + hook.set_item("name", &name)?; + hook.set_item("kwargs", entry.getattr("kwargs")?)?; + if let Some(fields) = python_hook_fields + && let Some(extra) = fields.get_item(&name)? + { + hook.call_method1("update", (extra,))?; + } + hooks.append(hook)?; + } + } + Ok(hooks) +} + +#[pyfunction] +#[pyo3(signature = (*, value, sql_fields, python_hook_fields = None))] +fn serialize_hook_entries<'py>( + value: &Bound<'py, PyAny>, + sql_fields: Vec, + python_hook_fields: Option>, +) -> PyResult> { + hook_entries(value, &sql_fields, python_hook_fields.as_ref()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(serialize_hook_entries, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index f70928f6fb..c0a9bd0a15 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod faults; pub(crate) mod guidance; pub(crate) mod header_declarations; pub(crate) mod header_values; +pub(crate) mod hook_entries; pub(crate) mod key_checks; pub(crate) mod locations; pub(crate) mod model_file_record; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs index 9bcc725c6d..b2817f23b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs @@ -4,9 +4,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::discovery::{ declaration_files, declaration_records, guidance, header_declarations, header_values, - key_checks, locations, model_records, project_files, project_validation, python_function_files, - python_hook_files, python_module_files, raised, scope_snapshot, snapshot, sql_test_records, - yaml_entries, yaml_records, + hook_entries, key_checks, locations, model_records, project_files, project_validation, + python_function_files, python_hook_files, python_module_files, raised, scope_snapshot, + snapshot, sql_test_records, yaml_entries, yaml_records, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -29,5 +29,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { yaml_entries::register(module)?; header_values::register(module)?; guidance::register(module)?; + hook_entries::register(module)?; Ok(()) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 05fa724c6f..830f8306c7 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2969,6 +2969,12 @@ def run_scenario_capture_command(*, request: ScenarioCaptureCommandRequest) -> i def sql_analysis_off_guidance( *, discovered_inputs: DiscoveredProjectInputs, purpose: str ) -> tuple[str, str] | None: ... +def serialize_hook_entries( + *, + value: object, + sql_fields: tuple[str, ...], + python_hook_fields: Mapping[str, Mapping[str, object]] | None = None, +) -> list[dict[str, object]]: ... def microbatch_guidance() -> tuple[str, str]: ... def explicit_references_help(*, allowed: str) -> str: ... def run_format_command( diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/hook_entries.py b/src/sqlbuild/compiler/discovery/_helpers/sql/hook_entries.py deleted file mode 100644 index e92f5e5db8..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/hook_entries.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Serialization of resolved model lifecycle hook entries.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry - - -def serialize_hook_entries_impl( - *, - value: object, - sql_fields: tuple[str, ...], - python_hook_fields: Mapping[str, Mapping[str, object]], -) -> list[dict[str, object]]: - """Serialize resolved hook entries, keeping only the requested SQL hook fields.""" - - if not isinstance(value, list | tuple): - return [] - hooks: list[dict[str, object]] = [] - entry: object - for entry in value: - if isinstance(entry, SqlHookEntry): - hooks.append(_serialize_sql_hook(entry=entry, sql_fields=sql_fields)) - elif isinstance(entry, PythonHookEntry): - python_hook: dict[str, object] = { - "type": "python", - "name": entry.name, - "kwargs": entry.kwargs, - } - python_hook.update(python_hook_fields.get(entry.name, {})) - hooks.append(python_hook) - return hooks - - -def _serialize_sql_hook(*, entry: SqlHookEntry, sql_fields: tuple[str, ...]) -> dict[str, object]: - field_values: dict[str, object | None] = { - "statement": entry.statement, - "name": entry.name, - "relative_path": ( - entry.relative_path.as_posix() if entry.relative_path is not None else None - ), - "definition_sql": entry.definition_sql, - "kwargs": entry.kwargs, - "description": entry.description, - } - hook: dict[str, object] = {"type": "sql"} - field_name: str - for field_name in sql_fields: - field_value: object | None = field_values[field_name] - if field_value is not None: - hook[field_name] = field_value - return hook diff --git a/src/sqlbuild/compiler/discovery/main/serialize_hook_entries.py b/src/sqlbuild/compiler/discovery/main/serialize_hook_entries.py deleted file mode 100644 index dcf654f9c1..0000000000 --- a/src/sqlbuild/compiler/discovery/main/serialize_hook_entries.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Model lifecycle hook serialization entrypoint.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.compiler.discovery._helpers.sql.hook_entries import serialize_hook_entries_impl - - -def serialize_hook_entries( - *, - value: object, - sql_fields: tuple[str, ...], - python_hook_fields: Mapping[str, Mapping[str, object]] | None = None, -) -> list[dict[str, object]]: - """Serialize resolved hook entries with an explicit SQL hook field allowlist.""" - - return serialize_hook_entries_impl( - value=value, sql_fields=sql_fields, python_hook_fields=python_hook_fields or {} - ) diff --git a/src/sqlbuild/compiler/planner/main/identity/version_identity_model_metadata.py b/src/sqlbuild/compiler/planner/main/identity/version_identity_model_metadata.py index 9c338abf61..0cc9e3b64b 100644 --- a/src/sqlbuild/compiler/planner/main/identity/version_identity_model_metadata.py +++ b/src/sqlbuild/compiler/planner/main/identity/version_identity_model_metadata.py @@ -4,9 +4,9 @@ from typing import Any +from sqlbuild._native import serialize_hook_entries from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.discovery.constants import SQL_HOOK_IDENTITY_FIELDS -from sqlbuild.compiler.discovery.main.serialize_hook_entries import serialize_hook_entries from sqlbuild.compiler.planner._helpers.identity.model_metadata import contract_output_signature from sqlbuild.compiler.planner.constants import ( MODEL_CUSTOM_CONFIG_KEY, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/main/test_serialize_hook_entries.py b/tests/unit/src/sqlbuild/compiler/discovery/main/test_serialize_hook_entries.py index 2df0c8979e..a5ba58ee08 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/main/test_serialize_hook_entries.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/main/test_serialize_hook_entries.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import serialize_hook_entries from sqlbuild.compiler.discovery.constants import SQL_HOOK_OUTPUT_FIELDS -from sqlbuild.compiler.discovery.main.serialize_hook_entries import serialize_hook_entries from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry from tests.unit.src.sqlbuild.compiler.discovery.main._test_types import ( SerializeHookEntriesTestCase, From 5ca3d255222a930faf1d9bd21fb62e81c25c5d03 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:16:22 +0100 Subject: [PATCH 06/24] refactor(discovery): drop unused discovery constants, models and errors --- src/sqlbuild/compiler/discovery/constants.py | 13 --------- src/sqlbuild/compiler/discovery/exceptions.py | 6 ----- src/sqlbuild/compiler/discovery/models.py | 27 ------------------- 3 files changed, 46 deletions(-) diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index 275bea262f..38ef4fb6c4 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -6,12 +6,9 @@ LOCAL_CONFIG_FILENAME: str = "sqlbuild_local.toml" LEGACY_PROJECT_CONFIG_FILENAME: str = "sqlbuild_project.yml" LEGACY_LOCAL_CONFIG_FILENAME: str = "sqlbuild_local.yml" -TOML_FILE_SUFFIX: str = ".toml" YAML_FILE_SUFFIXES: frozenset[str] = frozenset({".yml", ".yaml"}) PYTHON_INIT_MODULE_STEM: str = "__init__" -PYTHON_CACHE_DIRECTORY_NAME: str = "__pycache__" -LANGUAGE_PYTHON_ROOT_PART_COUNT: int = 2 PYTHON_NODE_ROOT: str = "python" SQL_FUNCTION_HEADER_KEYS: frozenset[str] = frozenset( {"arguments", "returns", "database", "schema", "tags", "description"} @@ -109,30 +106,20 @@ DLT_SOURCE_TYPE_REST_API: str = "rest_api" DLT_SOURCE_TYPE_FILESYSTEM: str = "filesystem" -CONFIG_CONCURRENCY_KEY: str = "concurrency" -LEGACY_CONFIG_CONCURRENCY_KEY: str = "max_concurrency" SQL_ANALYSIS_CONFIG_KEY: str = "sql_analysis" SQL_TEST_HEADER_KEYS: frozenset[str] = frozenset( {"name", "mode", "parameters", "cases", "cursor_start", "cursor_end", SQL_ANALYSIS_CONFIG_KEY} ) SQL_SCENARIO_HEADER_KEYS: frozenset[str] = frozenset({"description", "tags"}) -LEGACY_SQL_VALIDATION_CONFIG_KEY: str = "sql_validation" SQL_ANALYSIS_SETTING_KEY: str = "sql_analysis" -REQUIRE_SQL_ANALYSIS_SETTING_KEY: str = "require_sql_analysis" SETTINGS_SECTION: str = "settings" QUERY_CHANGE_TRACKING_SETTING_KEY: str = "query_change_tracking" -MICROBATCH_CONCURRENCY_SETTING_KEY: str = "microbatch_concurrency" REFERENCES_SECTION: str = "references" TABLE_PROMOTION_MODE_SETTING_KEY: str = "table_promotion_mode" STAGED_PROMOTION_MODE: str = "staged" DBT_SECTION: str = "dbt" DEFAULTS_SECTION: str = "defaults" DBT_PROJECT_DIR_KEY: str = "project_dir" -ENFORCE_EXPLICIT_REFERENCES_KEY: str = "enforce_explicit" -DBT_LEGACY_REUSE_FROM_CONFIG_KEY: str = "reuse_from" -DBT_PRODUCTION_REF_CONFIG_KEY: str = "production_ref" -DBT_DEFER_CLONE_CONFIG_KEY: str = "defer_clone_from" -DBT_REPLAY_ON_CHANGE_CONFIG_KEY: str = "replay_on_change" MODELS_DIRECTORY_NAME: str = "models" SQL_TESTS_OWNERSHIP_ROOT: str = "tests/unit" diff --git a/src/sqlbuild/compiler/discovery/exceptions.py b/src/sqlbuild/compiler/discovery/exceptions.py index 9fedbe5cf9..12ffcbe54f 100644 --- a/src/sqlbuild/compiler/discovery/exceptions.py +++ b/src/sqlbuild/compiler/discovery/exceptions.py @@ -27,12 +27,6 @@ class ModelSqlParseError(DiscoveryError): code: str = "D002" -class ModelHeaderSyntaxError(DiscoveryError): - """Raised when a MODEL(...) SQLBuild header cannot be parsed.""" - - code: str = "D002" - - class SqlTestParseError(DiscoveryError): """Raised when an authored SQL-native test file has an invalid shape.""" diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index c76212cd6c..17ee06a0c9 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -540,33 +540,6 @@ class DiscoveredPythonNodeFunctions: audit_factories: tuple[DiscoveredAuditFactory, ...] = field(default_factory=tuple) -@dataclass(frozen=True) -class DiscoveredDeclarationFiles: - """Project files discovery parses without importing any project Python.""" - - source_files: tuple[DiscoveredSourceFile, ...] - model_files: tuple[DiscoveredSqlModelFile, ...] - enum_files: tuple[DiscoveredEnumFile, ...] - constant_files: tuple[DiscoveredConstantFile, ...] - model_schema_files: tuple[DiscoveredModelSchemaFile, ...] - sql_function_files: tuple[DiscoveredSqlFunctionFile, ...] - sql_hook_files: tuple[DiscoveredSqlHookFile, ...] - python_function_files: tuple[DiscoveredPythonFunctionFile, ...] - schema_files: tuple[DiscoveredSchemaFile, ...] - seed_files: tuple[DiscoveredSeedFile, ...] - test_files: tuple[DiscoveredSqlTestFile, ...] - scenario_files: tuple[DiscoveredSqlScenarioFile, ...] - audit_files: tuple[DiscoveredAuditFile, ...] - macro_files: tuple[DiscoveredMacroFile, ...] - adapter_file: DiscoveredAdapterFile | None - native_session: NativeDiscoverySession | None = field(default=None, compare=False, repr=False) - """The native session that read these declaration files, for later native stages. - - `None` means no session: readers must use the Python objects. Any copy that changes a - discovered collection must set it to `None` so it never describes other files. - """ - - @dataclass(frozen=True) class DiscoveredProjectInputs: """All raw project inputs discovered from disk before semantic resolution.""" From 3f8253be99335ac02c430848cc251ac51390de6f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:32:38 +0100 Subject: [PATCH 07/24] refactor(discovery): validate project configuration natively --- .../sqlbuild-config/src/project/main/mod.rs | 1 + .../main/unsupported_target_warehouse.rs | 27 +++ .../src/project/tests/test_guidance.rs | 23 ++ .../_helpers/command_runtime/warehouses.rs | 2 +- .../compile_driver/config_validation.rs | 211 ++++++++++++++++++ .../_helpers/compile_driver/discovery.rs | 37 ++- .../bindings/_helpers/compile_driver/mod.rs | 1 + src/sqlbuild/_native.pyi | 4 +- ...pport.py => _session_warehouse_support.py} | 0 .../_helpers/validation/target_connections.py | 28 --- .../discovery/_helpers/yml/project.py | 54 ----- .../_helpers/yml/target_warehouses.py | 92 -------- .../main/_model_description_inputs.py | 4 +- .../discovery/main/runtime_extensions.py | 6 +- .../discovery/_helpers/_test_helpers.py | 4 +- .../compiler/discovery/_helpers/helpers.py | 14 ++ .../_helpers/test_statement_header_keys.py | 2 +- .../discovery/_helpers/test_target_diff.py | 8 +- .../_helpers/test_target_warehouses.py | 8 +- .../discovery/_helpers/test_yml_project.py | 6 +- 20 files changed, 312 insertions(+), 220 deletions(-) create mode 100644 crates/sqlbuild-config/src/project/main/unsupported_target_warehouse.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs rename src/sqlbuild/adapter/discovery/main/{session_warehouse_support.py => _session_warehouse_support.py} (100%) delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/validation/target_connections.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/yml/project.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/yml/target_warehouses.py diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs index e9a9a46956..6a2fc8fe75 100644 --- a/crates/sqlbuild-config/src/project/main/mod.rs +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -5,3 +5,4 @@ pub mod local_sql_analysis_failure; pub mod microbatch_guidance; pub mod resolve_config_file; pub mod sql_analysis_off_guidance; +pub mod unsupported_target_warehouse; diff --git a/crates/sqlbuild-config/src/project/main/unsupported_target_warehouse.rs b/crates/sqlbuild-config/src/project/main/unsupported_target_warehouse.rs new file mode 100644 index 0000000000..6be579c5ca --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/unsupported_target_warehouse.rs @@ -0,0 +1,27 @@ +//! The error for a target warehouse group under an adapter without session warehouses. + +use crate::project::_helpers::help::{Setting, SettingValue, setting_note}; + +/// The message and help naming `[targets..warehouses] = ` in `file_name`. +#[must_use] +pub fn unsupported_target_warehouse( + (file_path, file_name): (&str, &str), + adapter_name: &str, + (target_name, group, value): (&str, &str, &str), +) -> (String, String) { + let section: String = format!("targets.{target_name}.warehouses"); + let note: String = setting_note(&Setting { + file_name, + section: §ion, + key: group, + value: SettingValue::Text(value), + }); + ( + format!( + "{file_path} [{section}] selects a {group} warehouse, but the '{adapter_name}' adapter has no warehouse to select; {note}" + ), + format!( + "remove the [{section}] section from {file_name}; command-group warehouses need an adapter with a session warehouse, such as snowflake or an adapter extending it" + ), + ) +} diff --git a/crates/sqlbuild-config/src/project/tests/test_guidance.rs b/crates/sqlbuild-config/src/project/tests/test_guidance.rs index b9290f28af..fe68a81e20 100644 --- a/crates/sqlbuild-config/src/project/tests/test_guidance.rs +++ b/crates/sqlbuild-config/src/project/tests/test_guidance.rs @@ -35,3 +35,26 @@ fn given_setting_guidance_when_rendering_then_text_matches_python() { ); } } + +#[test] +fn given_target_warehouse_without_session_warehouses_when_rejecting_then_text_matches_python() { + let test_cases = [GuidanceTestCase { + description: "a quoted local query warehouse", + rendered: crate::project::main::unsupported_target_warehouse::unsupported_target_warehouse( + ("/p/sqlbuild_local.toml", "sqlbuild_local.toml"), + "duckdb", + ("dev", "query", "WH \"x\""), + ), + expected_note: "/p/sqlbuild_local.toml [targets.dev.warehouses] selects a query warehouse, but the 'duckdb' adapter has no warehouse to select; sqlbuild_local.toml sets [targets.dev.warehouses] query = \"WH \\\"x\\\"\"", + expected_help: "remove the [targets.dev.warehouses] section from sqlbuild_local.toml; command-group warehouses need an adapter with a session warehouse, such as snowflake or an adapter extending it", + }]; + + for test_case in test_cases { + assert_eq!( + (test_case.rendered.0.as_str(), test_case.rendered.1.as_str()), + (test_case.expected_note, test_case.expected_help), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index 0ac831302d..daff68d0bd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -28,7 +28,7 @@ const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; const SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; -const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main.session_warehouse_support"; +const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main._session_warehouse_support"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; const EXPAND_TEMPLATE_MODULE: &str = "sqlbuild.compiler.compile.main.expand_template_data"; const RESOLVE_OPTIONS: (&[&str], &[&str]) = ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs new file mode 100644 index 0000000000..fee36f43b6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs @@ -0,0 +1,211 @@ +//! Validate loaded project and local configuration before discovery reads project files. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyString, PyTuple}; +use sqlbuild_config::project::main::unsupported_target_warehouse::unsupported_target_warehouse; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const SPEC_ERRORS_MODULE: &str = "sqlbuild.spec.contracts.exceptions"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main._session_warehouse_support"; +const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; +const SQL_ANALYSIS_KEY: &str = "sql_analysis"; +/// `WarehouseGroup` members in declaration order. +const WAREHOUSE_GROUPS: [&str; 2] = ["build", "query"]; + +/// The loaded configuration of one project. +pub(crate) struct ProjectConfiguration<'a, 'py> { + pub(crate) project_dir: &'a Bound<'py, PyAny>, + pub(crate) project_config: &'a Bound<'py, PyAny>, + pub(crate) local_config: &'a Bound<'py, PyAny>, +} + +fn project_config_error(py: Python<'_>, message: &str, help: Option<&str>) -> PyErr { + let raised = (|| -> PyResult> { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("help", help)?; + imported(py, DISCOVERY_ERRORS_MODULE, "ProjectConfigError")? + .call((message,), Some(&keywords)) + })(); + match raised { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +/// The target names of one configuration's `targets` mapping. +fn target_names(config: &Bound<'_, PyAny>) -> PyResult> { + config + .getattr("targets")? + .try_iter()? + .map(|name| name?.extract()) + .collect() +} + +/// Reject a local SQL analysis opt-out the project forbids. +fn reject_local_sql_analysis_opt_out(config: &ProjectConfiguration<'_, '_>) -> PyResult<()> { + let py: Python<'_> = config.project_dir.py(); + let local_settings: Bound<'_, PyAny> = config.local_config.getattr("settings")?; + let flags: Bound<'_, PyTuple> = PyTuple::new( + py, + [ + config + .project_config + .getattr("settings")? + .getattr("require_sql_analysis")?, + config + .local_config + .getattr("setting_overrides")? + .contains(SQL_ANALYSIS_KEY) + .map(|overridden| PyBool::new(py, overridden).to_owned().into_any())?, + local_settings.getattr(SQL_ANALYSIS_KEY)?, + ], + )?; + imported(py, NATIVE_MODULE, "validate_local_sql_analysis")? + .call1((config.project_dir, flags))?; + Ok(()) +} + +/// The first target, by name, that selects a command-group warehouse. +fn first_configured_group(config: &Bound<'_, PyAny>) -> PyResult> { + let targets: Bound<'_, PyAny> = config.getattr("targets")?; + let mut names: Vec = target_names(config)?; + names.sort(); + for name in names { + let warehouses: Bound<'_, PyAny> = targets.get_item(&name)?.getattr("warehouses")?; + for group in WAREHOUSE_GROUPS { + let value: Bound<'_, PyAny> = warehouses.getattr(group)?; + if !value.is_none() { + return Ok(Some((name, group.to_owned(), value.extract()?))); + } + } + } + Ok(None) +} + +fn session_warehouse_support( + py: Python<'_>, + adapter_name: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult { + call_with( + &imported( + py, + SESSION_WAREHOUSE_MODULE, + "adapter_supports_session_warehouse", + )?, + &[ + ("adapter_name", adapter_name.clone()), + ("project_dir", project_dir.clone()), + ], + )? + .is_truthy() +} + +fn reject_group( + py: Python<'_>, + (local, adapter_name): (bool, &Bound<'_, PyAny>), + project_dir: &Bound<'_, PyAny>, + (target, group, value): (String, String, String), +) -> PyErr { + let raised = (|| -> PyResult { + let file_path: String = imported(py, NATIVE_MODULE, "resolve_config_file_path")? + .call1((project_dir, local))? + .str()? + .extract()?; + let file_name: &str = if local { + LOCAL_CONFIG_FILENAME + } else { + PROJECT_CONFIG_FILENAME + }; + let (message, help) = unsupported_target_warehouse( + (&file_path, file_name), + &adapter_name.extract::()?, + (&target, &group, &value), + ); + Ok(project_config_error(py, &message, Some(&help))) + })(); + raised.unwrap_or_else(|failure| failure) +} + +/// Reject command-group warehouses under adapters without a session warehouse. +fn reject_unsupported_target_warehouses(config: &ProjectConfiguration<'_, '_>) -> PyResult<()> { + let py: Python<'_> = config.project_dir.py(); + if let Some(group) = first_configured_group(config.project_config)? { + let adapter_name: Bound<'_, PyAny> = config.project_config.getattr("adapter")?; + if !session_warehouse_support(py, &adapter_name, config.project_dir)? { + return Err(reject_group( + py, + (false, &adapter_name), + config.project_dir, + group, + )); + } + } + let Some(group) = first_configured_group(config.local_config)? else { + return Ok(()); + }; + let adapter_name: Bound<'_, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", config.project_config.clone()), + ("local_config", config.local_config.clone()), + ], + )?; + if session_warehouse_support(py, &adapter_name, config.project_dir)? { + return Ok(()); + } + Err(reject_group( + py, + (true, &adapter_name), + config.project_dir, + group, + )) +} + +/// Resolve every configured target so invalid connections fail discovery. +fn reject_invalid_target_connections(config: &ProjectConfiguration<'_, '_>) -> PyResult<()> { + let py: Python<'_> = config.project_dir.py(); + let mut names: Vec = target_names(config.project_config)?; + names.extend(target_names(config.local_config)?); + names.sort(); + names.dedup(); + let resolve: Bound<'_, PyAny> = imported(py, TARGET_CONFIG_MODULE, "resolve_target_config")?; + let spec_error: Bound<'_, PyAny> = imported(py, SPEC_ERRORS_MODULE, "SpecConfigError")?; + for name in names { + let resolved: PyResult> = call_with( + &resolve, + &[ + ("project_config", config.project_config.clone()), + ("local_config", config.local_config.clone()), + ("target_name", PyString::new(py, &name).into_any()), + ], + ); + let Err(error) = resolved else { + continue; + }; + if !error.is_instance(py, &spec_error) { + return Err(error); + } + let config_path: Bound<'_, PyAny> = config.project_dir.div(PROJECT_CONFIG_FILENAME)?; + let message: String = format!("{} {}", config_path.str()?, error.value(py).str()?); + let raised: PyErr = project_config_error(py, &message, None); + raised.set_cause(py, Some(error)); + return Err(raised); + } + Ok(()) +} + +/// Validate the SQL analysis policy, target warehouses and target connections, in that order. +pub(crate) fn reject_invalid_project_configuration( + config: &ProjectConfiguration<'_, '_>, +) -> PyResult<()> { + reject_local_sql_analysis_opt_out(config)?; + reject_unsupported_target_warehouses(config)?; + reject_invalid_target_connections(config) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index 325e0605de..3db998718a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -4,6 +4,9 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyCFunction, PyDict, PySet, PySetMethods, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::compile_driver::config_validation::{ + ProjectConfiguration, reject_invalid_project_configuration, +}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, within, }; @@ -28,10 +31,8 @@ use crate::bindings::_helpers::discovery::yaml_records::{ const FRONTIER_MODULE: &str = "sqlbuild._native"; const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; -const PROJECT_YAML_MODULE: &str = "sqlbuild.compiler.discovery._helpers.yml.project"; -const TARGET_CONNECTIONS_MODULE: &str = - "sqlbuild.compiler.discovery._helpers.validation.target_connections"; const VALIDATION_MODULE: &str = "sqlbuild.compiler.discovery._helpers.validation.discovery"; +const NATIVE_MODULE: &str = "sqlbuild._native"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const EVENT_EXPORTERS_MODULE: &str = "sqlbuild.runtime.event_exporting.main._configure_discovered_event_exporters"; @@ -210,27 +211,15 @@ fn assembled_inputs<'py>( request: &DiscoveryRequest, ) -> PyResult> { let project_dir: &Bound<'py, PyAny> = request.project_dir.bind(py); - let directory: [(&str, Bound<'py, PyAny>); 1] = [("project_dir", project_dir.clone())]; - let project_config: Bound<'py, PyAny> = call_with( - &imported(py, PROJECT_YAML_MODULE, "load_project_config")?, - &directory, - )?; - let local_config: Bound<'py, PyAny> = call_with( - &imported(py, PROJECT_YAML_MODULE, "load_local_config")?, - &directory, - )?; - let configured: [(&str, Bound<'py, PyAny>); 3] = [ - ("project_dir", project_dir.clone()), - ("project_config", project_config.clone()), - ("local_config", local_config.clone()), - ]; - for (module, name) in [ - (PROJECT_YAML_MODULE, "validate_local_sql_analysis_policy"), - (PROJECT_YAML_MODULE, "validate_target_warehouses_adapter"), - (TARGET_CONNECTIONS_MODULE, "validate_target_connections"), - ] { - call_with(&imported(py, module, name)?, &configured)?; - } + let project_config: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "load_project_config_file")?.call1((project_dir,))?; + let local_config: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "load_local_config_file")?.call1((project_dir,))?; + reject_invalid_project_configuration(&ProjectConfiguration { + project_dir, + project_config: &project_config, + local_config: &local_config, + })?; reject_unsupported_python_paths(project_dir)?; let sql_analysis_override: &Bound<'py, PyAny> = request.sql_analysis_enabled_override.bind(py); let sql_analysis_enabled: Bound<'py, PyAny> = if !sql_analysis_override.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index 4e7a62bd54..7ebf3fd7ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod assembly_checks; pub(crate) mod assembly_models; pub(crate) mod assembly_project; pub(crate) mod command; +pub(crate) mod config_validation; pub(crate) mod deferral; pub(crate) mod diff; pub(crate) mod discovery; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 830f8306c7..ffb37dbfac 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2439,8 +2439,8 @@ def find_python_string_locations( def resource_identity_suggestion( name: str, private_identity: bool, / ) -> tuple[str, str] | None: ... -def load_project_config_file(project_dir: Path, /) -> object: ... -def load_local_config_file(project_dir: Path, /) -> object: ... +def load_project_config_file(project_dir: Path, /) -> ProjectConfig: ... +def load_local_config_file(project_dir: Path, /) -> LocalConfig: ... def resolve_config_file_path(project_dir: Path, local: bool, /) -> Path: ... def validate_local_sql_analysis(project_dir: Path, flags: tuple[bool, bool, bool], /) -> None: ... def sql_test_cte_span( diff --git a/src/sqlbuild/adapter/discovery/main/session_warehouse_support.py b/src/sqlbuild/adapter/discovery/main/_session_warehouse_support.py similarity index 100% rename from src/sqlbuild/adapter/discovery/main/session_warehouse_support.py rename to src/sqlbuild/adapter/discovery/main/_session_warehouse_support.py diff --git a/src/sqlbuild/compiler/discovery/_helpers/validation/target_connections.py b/src/sqlbuild/compiler/discovery/_helpers/validation/target_connections.py deleted file mode 100644 index fd6a20692e..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/validation/target_connections.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Offline validation of each target's connection selection.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery.constants import PROJECT_CONFIG_FILENAME -from sqlbuild.compiler.discovery.exceptions import ProjectConfigError -from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -def validate_target_connections( - *, project_dir: Path, project_config: ProjectConfig, local_config: LocalConfig -) -> None: - """Reject targets whose connection cannot be chosen among several named connections.""" - - target_name: str - for target_name in sorted({*project_config.targets, *local_config.targets}): - try: - _ = resolve_target_config( - project_config=project_config, - local_config=local_config, - target_name=target_name, - ) - except SpecConfigError as error: - raise ProjectConfigError(f"{project_dir / PROJECT_CONFIG_FILENAME} {error}") from error diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py deleted file mode 100644 index b885047716..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Project config loading helpers.""" - -from __future__ import annotations - -from pathlib import Path -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.yml.target_warehouses import ( - reject_unsupported_target_warehouses, -) -from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_CONFIG_KEY -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -def load_project_config(*, project_dir: Path) -> ProjectConfig: - """Load project config from the given project directory.""" - - return cast(ProjectConfig, _native.load_project_config_file(project_dir)) - - -def load_local_config(*, project_dir: Path) -> LocalConfig: - """Load local config if present.""" - - return cast(LocalConfig, _native.load_local_config_file(project_dir)) - - -def validate_local_sql_analysis_policy( - *, project_dir: Path, project_config: ProjectConfig, local_config: LocalConfig -) -> None: - """Reject a local project-wide SQL analysis opt-out when the project requires analysis.""" - - _native.validate_local_sql_analysis( - project_dir, - ( - project_config.settings.require_sql_analysis, - SQL_ANALYSIS_CONFIG_KEY in local_config.setting_overrides, - local_config.settings.sql_analysis, - ), - ) - - -def validate_target_warehouses_adapter( - *, project_dir: Path, project_config: ProjectConfig, local_config: LocalConfig -) -> None: - """Reject command-group warehouses for adapters without a session warehouse.""" - - reject_unsupported_target_warehouses( - project_dir=project_dir, - project_config=project_config, - local_config=local_config, - project_config_path=_native.resolve_config_file_path(project_dir, False), - local_config_path=_native.resolve_config_file_path(project_dir, True), - ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/target_warehouses.py b/src/sqlbuild/compiler/discovery/_helpers/yml/target_warehouses.py deleted file mode 100644 index cdd96a1c37..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/target_warehouses.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Adapter validation for per-target command-group warehouses.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.adapter.discovery.main.session_warehouse_support import ( - adapter_supports_session_warehouse, -) -from sqlbuild.compiler.discovery.constants import LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME -from sqlbuild.compiler.discovery.exceptions import ProjectConfigError -from sqlbuild.errors.setting_help.main.setting_note import setting_note -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, TargetWarehousesConfig -from sqlbuild.spec.contracts.types import WarehouseGroup - - -def reject_unsupported_target_warehouses( - *, - project_dir: Path, - project_config: ProjectConfig, - local_config: LocalConfig, - project_config_path: Path, - local_config_path: Path, -) -> None: - """Check shared groups against the shared adapter and local groups against the effective one.""" - - project_group: tuple[str, WarehouseGroup, str] | None = _first_configured_group( - {name: target.warehouses for name, target in project_config.targets.items()} - ) - if project_group is not None and not adapter_supports_session_warehouse( - adapter_name=project_config.adapter, project_dir=project_dir - ): - _raise_unsupported_target_warehouse( - adapter_name=project_config.adapter, - file_path=project_config_path, - file_name=PROJECT_CONFIG_FILENAME, - configured=project_group, - ) - local_group: tuple[str, WarehouseGroup, str] | None = _first_configured_group( - {name: target.warehouses for name, target in local_config.targets.items()} - ) - if local_group is None: - return - effective_adapter: str = resolve_effective_adapter_name( - project_config=project_config, local_config=local_config - ) - if not adapter_supports_session_warehouse( - adapter_name=effective_adapter, project_dir=project_dir - ): - _raise_unsupported_target_warehouse( - adapter_name=effective_adapter, - file_path=local_config_path, - file_name=LOCAL_CONFIG_FILENAME, - configured=local_group, - ) - - -def _first_configured_group( - warehouses_by_target: dict[str, TargetWarehousesConfig], -) -> tuple[str, WarehouseGroup, str] | None: - target_name: str - warehouses: TargetWarehousesConfig - for target_name, warehouses in sorted(warehouses_by_target.items()): - group: WarehouseGroup - for group in WarehouseGroup: - value: str | None = warehouses.for_group(group) - if value is not None: - return target_name, group, value - return None - - -def _raise_unsupported_target_warehouse( - *, - adapter_name: str, - file_path: Path, - file_name: str, - configured: tuple[str, WarehouseGroup, str], -) -> None: - target_name, group, value = configured - section: str = f"targets.{target_name}.warehouses" - raise ProjectConfigError( - f"{file_path} [{section}] selects a {group.value} warehouse, but the " - f"'{adapter_name}' adapter has no warehouse to select; " - + setting_note(file_name=file_name, section=section, key=group.value, value=value), - help=( - f"remove the [{section}] section from {file_name}; command-group warehouses need an " - "adapter with a session warehouse, such as snowflake or an adapter extending it" - ), - ) diff --git a/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py b/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py index c5cdea0176..bd2649e7d0 100644 --- a/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py +++ b/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py @@ -8,8 +8,8 @@ directory_snapshot_scope, discover_model_files, discover_model_schema_files, + load_project_config_file, ) -from sqlbuild.compiler.discovery._helpers.yml.project import load_project_config from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryFileFault from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig @@ -17,7 +17,7 @@ def discover_model_description_inputs(*, project_dir: Path) -> DiscoveredProjectInputs: """Load project config, model files and model schemas without importing project Python.""" - project_config: ProjectConfig = load_project_config(project_dir=project_dir) + project_config: ProjectConfig = load_project_config_file(project_dir) with directory_snapshot_scope(project_dir=project_dir): return DiscoveredProjectInputs( project_config=project_config, diff --git a/src/sqlbuild/compiler/discovery/main/runtime_extensions.py b/src/sqlbuild/compiler/discovery/main/runtime_extensions.py index a7cd51954d..6af1f392ea 100644 --- a/src/sqlbuild/compiler/discovery/main/runtime_extensions.py +++ b/src/sqlbuild/compiler/discovery/main/runtime_extensions.py @@ -3,6 +3,7 @@ from dataclasses import replace from pathlib import Path +import sqlbuild._native as _native from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( bind_command_output_sink_declarations, discover_command_output_sink_declarations, @@ -12,7 +13,6 @@ discover_event_exporter_declarations, discover_provider_classes, ) -from sqlbuild.compiler.discovery._helpers.yml.project import load_project_config from sqlbuild.compiler.discovery.exceptions import ProjectConfigError from sqlbuild.compiler.discovery.models import ( DiscoveredCommandOutputSink, @@ -45,7 +45,9 @@ def discover_runtime_extensions(*, project_dir: Path) -> DiscoveredRuntimeExtens and not (project_dir / "sqlbuild_project.toml").exists() ): return DiscoveredRuntimeExtensions() - config: LifecycleEventSinksConfig = load_project_config(project_dir=project_dir).sinks.lifecycle + config: LifecycleEventSinksConfig = _native.load_project_config_file( + project_dir + ).sinks.lifecycle declaration_names: frozenset[str] = frozenset(item.name for item in declarations) unknown_names: frozenset[str] = frozenset(config.named) - declaration_names if unknown_names: diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py index 24039af664..db5f550b6e 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py @@ -24,11 +24,11 @@ discover_sql_hook_files, discover_test_files, ) -from sqlbuild.compiler.discovery._helpers.yml.project import ( +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import ( load_local_config, load_project_config, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs # Test-only reference grammar of the MODEL header the native matcher replaced. _REFERENCE_MODEL_HEADER_PATTERN: re.Pattern[str] = re.compile( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py index 60d42d0a87..8388c04f1b 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py @@ -2,6 +2,7 @@ from pathlib import Path from typing import cast +from sqlbuild import _native from sqlbuild._native import ( discover_audit_files, discover_constant_files, @@ -21,6 +22,7 @@ DiscoveredSqlTestCase, SqlTestParameterDeclaration, ) +from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( LoadProjectConfigTestCase, ) @@ -179,3 +181,15 @@ def parse_statement_header_file(*, statement: str, contents: str, file_path: Pat """Parse one authored statement file with the discovery parser for its header kind.""" return _STATEMENT_HEADER_PARSERS[statement](contents=contents, file_path=file_path) + + +def load_project_config(*, project_dir: Path) -> ProjectConfig: + """Load the shared project configuration through the native reader.""" + + return _native.load_project_config_file(project_dir) + + +def load_local_config(*, project_dir: Path) -> LocalConfig: + """Load the local configuration overrides through the native reader.""" + + return _native.load_local_config_file(project_dir) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_statement_header_keys.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_statement_header_keys.py index c3e4a1b849..322f3db4bc 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_statement_header_keys.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_statement_header_keys.py @@ -6,12 +6,12 @@ import pytest -from sqlbuild.compiler.discovery._helpers.yml.project import load_project_config from sqlbuild.compiler.discovery.exceptions import DiscoveryError, ProjectConfigError from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( StatementHeaderKeysTestCase, ) from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import ( + load_project_config, parse_statement_header_file, statement_header_file_path, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_diff.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_diff.py index b0c6b96a8c..d0f29c74d3 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_diff.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_diff.py @@ -6,16 +6,16 @@ import pytest -from sqlbuild.compiler.discovery._helpers.yml.project import ( - load_local_config, - load_project_config, -) from sqlbuild.compiler.discovery.exceptions import ProjectConfigError from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( TargetDiffConfigErrorTestCase, TargetDiffConfigTestCase, ) +from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import ( + load_local_config, + load_project_config, +) _PROJECT: str = 'name = "shop"\nadapter = "duckdb"\n\n[targets.dev]\nschema = "dev"\n' diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py index 340f686c7b..ae5088970b 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py @@ -7,10 +7,6 @@ import pytest from sqlbuild._native import discover_project_inputs -from sqlbuild.compiler.discovery._helpers.yml.project import ( - load_local_config, - load_project_config, -) from sqlbuild.compiler.discovery.exceptions import ProjectConfigError from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.models import TargetWarehousesConfig @@ -19,6 +15,10 @@ TargetWarehousesConfigTestCase, TargetWarehousesDiscoveryTestCase, ) +from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import ( + load_local_config, + load_project_config, +) _SNOWFLAKE_PROJECT: str = """ name = "shop" diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py index 6f66a4c817..4b21e81273 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py @@ -6,10 +6,6 @@ import pytest -from sqlbuild.compiler.discovery._helpers.yml.project import ( - load_local_config, - load_project_config, -) from sqlbuild.compiler.discovery.exceptions import ProjectConfigError from sqlbuild.runtime.event_exporting.types import LifecycleEventKind from sqlbuild.spec.contracts.models import ( @@ -52,6 +48,8 @@ TooDeepProjectConnectionTestCase, ) from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import ( + load_local_config, + load_project_config, write_project_config_test_files, ) From 2833a5e3926ee84b8d3895fae060c7045afb9c28 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:55:00 +0100 Subject: [PATCH 08/24] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 290d4509c5..154d97d82b 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 69877 +remaining_lines = 67450 [[stays]] path = "executor" From ea0192f332d301c653b5611c0ac94812959bf513 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:54:43 +0100 Subject: [PATCH 09/24] feat: render planner display fragments natively --- crates/sqlbuild-core/src/lib.rs | 1 + .../src/plan_display/_helpers/labels.rs | 36 +++ .../src/plan_display/_helpers/mod.rs | 1 + .../src/plan_display/constants.rs | 26 ++ .../sqlbuild-core/src/plan_display/errors.rs | 5 + .../plan_display/main/cursor_bound_display.rs | 25 ++ .../main/effective_microbatch_batch_size.rs | 19 ++ .../plan_display/main/format_no_input_rows.rs | 7 + .../main/format_waiting_on_empty_inputs.rs | 15 + .../src/plan_display/main/mod.rs | 7 + .../main/model_execution_annotation.rs | 22 ++ .../main/model_materialization_label.rs | 43 +++ .../plan_display/main/model_resource_type.rs | 16 + crates/sqlbuild-core/src/plan_display/mod.rs | 10 + .../sqlbuild-core/src/plan_display/models.rs | 25 ++ .../sqlbuild-core/src/plan_display/tests.rs | 6 + .../src/plan_display/tests/helpers.rs | 10 + .../plan_display/tests/test_plan_display.rs | 209 +++++++++++++ .../src/plan_display/tests/test_types.rs | 31 ++ .../src/bindings/_helpers/cursor/scalars.rs | 6 +- .../_helpers/plan_output/cursor_details.rs | 36 +-- .../bindings/_helpers/plan_output/display.rs | 284 ++++++++++++++++++ .../src/bindings/_helpers/plan_output/mod.rs | 1 + .../_helpers/plan_output/plan_text.rs | 69 ++--- .../_helpers/plan_output/registration.rs | 3 +- src/sqlbuild/_native.pyi | 31 +- .../classes/build_progress_callbacks.py | 20 +- .../_helpers/output/inclusive_cursor_end.py | 86 ------ .../_helpers/output/materialization_labels.py | 94 ------ .../_helpers/output/microbatch_bounds.py | 5 +- .../_helpers/output/microbatch_count.py | 4 +- .../planner/_helpers/output/plan_entry.py | 8 +- .../planner/_helpers/resolve/cursor.py | 13 - .../planner/_helpers/resolve/resolve.py | 4 +- .../cursor_window/no_input_rows_display.py | 13 - .../waiting_on_empty_inputs_display.py | 13 - .../execution/_model_materialization_label.py | 14 - .../execution/aligned_timestamp_bounds.py | 28 -- .../main/execution/cursor_bound_display.py | 13 - .../effective_microbatch_batch_size.py | 15 - .../main/execution/inclusive_cursor_end.py | 13 - .../execution/materialization_type_display.py | 14 - .../execution/model_execution_annotation.py | 14 - .../main/execution/model_resource_type.py | 14 - .../run/_helpers/execution/results.py | 4 +- .../_helpers/materializations/microbatch.py | 16 +- .../planner/main/test_cursor_bound_display.py | 2 +- .../test_effective_microbatch_batch_size.py | 4 +- .../planner/main/test_inclusive_cursor_end.py | 2 +- 49 files changed, 886 insertions(+), 471 deletions(-) create mode 100644 crates/sqlbuild-core/src/plan_display/_helpers/labels.rs create mode 100644 crates/sqlbuild-core/src/plan_display/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/plan_display/constants.rs create mode 100644 crates/sqlbuild-core/src/plan_display/errors.rs create mode 100644 crates/sqlbuild-core/src/plan_display/main/cursor_bound_display.rs create mode 100644 crates/sqlbuild-core/src/plan_display/main/effective_microbatch_batch_size.rs create mode 100644 crates/sqlbuild-core/src/plan_display/main/format_no_input_rows.rs create mode 100644 crates/sqlbuild-core/src/plan_display/main/format_waiting_on_empty_inputs.rs create mode 100644 crates/sqlbuild-core/src/plan_display/main/mod.rs create mode 100644 crates/sqlbuild-core/src/plan_display/main/model_execution_annotation.rs create mode 100644 crates/sqlbuild-core/src/plan_display/main/model_materialization_label.rs create mode 100644 crates/sqlbuild-core/src/plan_display/main/model_resource_type.rs create mode 100644 crates/sqlbuild-core/src/plan_display/mod.rs create mode 100644 crates/sqlbuild-core/src/plan_display/models.rs create mode 100644 crates/sqlbuild-core/src/plan_display/tests.rs create mode 100644 crates/sqlbuild-core/src/plan_display/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs create mode 100644 crates/sqlbuild-core/src/plan_display/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/output/inclusive_cursor_end.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/output/materialization_labels.py delete mode 100644 src/sqlbuild/compiler/planner/main/cursor_window/no_input_rows_display.py delete mode 100644 src/sqlbuild/compiler/planner/main/cursor_window/waiting_on_empty_inputs_display.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/_model_materialization_label.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/aligned_timestamp_bounds.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/cursor_bound_display.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/effective_microbatch_batch_size.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/inclusive_cursor_end.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/materialization_type_display.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/model_execution_annotation.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/model_resource_type.py diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index bc9af8ff56..1bc27fe670 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -22,6 +22,7 @@ pub mod json; pub mod load_command; pub mod models; pub mod panics; +pub mod plan_display; pub mod posix_path; pub mod progress; pub mod query_command; diff --git a/crates/sqlbuild-core/src/plan_display/_helpers/labels.rs b/crates/sqlbuild-core/src/plan_display/_helpers/labels.rs new file mode 100644 index 0000000000..0fb8256968 --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/_helpers/labels.rs @@ -0,0 +1,36 @@ +//! The snapshot label shared by plan summaries and progress rows. + +use crate::plan_display::constants::SNAPSHOT; +use crate::plan_display::models::ModelLabelFacts; + +/// The snapshot strategy and historical input, optionally as `snapshot (
)`. +pub(crate) fn snapshot_label(entry: &ModelLabelFacts, include_prefix: bool) -> String { + let mut parts: Vec = Vec::new(); + if let Some(strategy) = entry + .snapshot_strategy + .as_deref() + .filter(|strategy| !strategy.is_empty()) + { + parts.push(strategy.to_owned()); + } + if entry.historical { + let input: &str = entry + .historical_input + .as_deref() + .filter(|input| !input.is_empty()) + .unwrap_or(SNAPSHOT); + parts.push(format!("historical {input}")); + } + if parts.is_empty() { + return if include_prefix { + SNAPSHOT.to_owned() + } else { + String::new() + }; + } + let label: String = parts.join(", "); + if include_prefix { + return format!("snapshot ({label})"); + } + label +} diff --git a/crates/sqlbuild-core/src/plan_display/_helpers/mod.rs b/crates/sqlbuild-core/src/plan_display/_helpers/mod.rs new file mode 100644 index 0000000000..3715888d75 --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod labels; diff --git a/crates/sqlbuild-core/src/plan_display/constants.rs b/crates/sqlbuild-core/src/plan_display/constants.rs new file mode 100644 index 0000000000..e771ddcad6 --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/constants.rs @@ -0,0 +1,26 @@ +//! Materialization, action and cursor vocabulary the plan display reads. + +pub(crate) const VIEW: &str = "view"; +pub(crate) const TABLE: &str = "table"; +pub(crate) const INCREMENTAL: &str = "incremental"; +pub(crate) const SNAPSHOT: &str = "snapshot"; +pub(crate) const CUSTOM: &str = "custom"; +pub(crate) const MICROBATCH_MODE: &str = "microbatch"; +pub(crate) const INCREMENTAL_ACTIONS: [&str; 3] = [ + "incremental_append", + "incremental_delete_insert", + "incremental_merge", +]; +pub const INTEGER_CURSOR: &str = "integer"; +pub const TIMESTAMP_CURSOR: &str = "timestamp"; +pub const DEFAULT_CURSOR_GRAIN: &str = "second"; +pub(crate) const WHOLE_DAY_CURSOR_GRAINS: [&str; 3] = ["day", "month", "year"]; +pub(crate) const EFFECTIVE_BATCH_SIZE_TOKEN: &str = "effective"; +pub(crate) const GRAIN_BATCH_SIZE: [(&str, &str); 6] = [ + ("second", "1s"), + ("minute", "1m"), + ("hour", "1h"), + ("day", "1d"), + ("month", "1mo"), + ("year", "1y"), +]; diff --git a/crates/sqlbuild-core/src/plan_display/errors.rs b/crates/sqlbuild-core/src/plan_display/errors.rs new file mode 100644 index 0000000000..638deb1ce6 --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/errors.rs @@ -0,0 +1,5 @@ +//! Plan display errors. + +/// A cursor grain outside the known grains. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct UnknownCursorGrain(pub String); diff --git a/crates/sqlbuild-core/src/plan_display/main/cursor_bound_display.rs b/crates/sqlbuild-core/src/plan_display/main/cursor_bound_display.rs new file mode 100644 index 0000000000..aceb8746ba --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/cursor_bound_display.rs @@ -0,0 +1,25 @@ +//! A cursor bound as plans and progress rows show it. + +use crate::plan_display::constants::{INTEGER_CURSOR, WHOLE_DAY_CURSOR_GRAINS}; +use crate::plan_display::models::ParsedBound; + +/// The value, collapsed to its date when a whole-day grain bound falls at midnight. +#[must_use] +pub fn cursor_bound_display( + value: &str, + (cursor_type, cursor_grain): (Option<&str>, Option<&str>), + parsed: Option<&ParsedBound>, +) -> String { + if cursor_type == Some(INTEGER_CURSOR) { + return value.to_owned(); + } + match parsed { + Some(bound) + if bound.midnight + && cursor_grain.is_some_and(|grain| WHOLE_DAY_CURSOR_GRAINS.contains(&grain)) => + { + bound.date.clone() + } + _ => value.to_owned(), + } +} diff --git a/crates/sqlbuild-core/src/plan_display/main/effective_microbatch_batch_size.rs b/crates/sqlbuild-core/src/plan_display/main/effective_microbatch_batch_size.rs new file mode 100644 index 0000000000..eadf5b09e4 --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/effective_microbatch_batch_size.rs @@ -0,0 +1,19 @@ +//! The microbatch size after cursor-grain propagation. + +use crate::plan_display::constants::{EFFECTIVE_BATCH_SIZE_TOKEN, GRAIN_BATCH_SIZE}; +use crate::plan_display::errors::UnknownCursorGrain; + +/// The declared size, or the grain's single-unit size for the `effective` token; an unknown grain is an error. +pub fn effective_microbatch_batch_size( + batch_size: &str, + effective_grain: &str, +) -> Result { + if batch_size != EFFECTIVE_BATCH_SIZE_TOKEN { + return Ok(batch_size.to_owned()); + } + GRAIN_BATCH_SIZE + .iter() + .find(|(grain, _)| *grain == effective_grain) + .map(|(_, size)| (*size).to_owned()) + .ok_or_else(|| UnknownCursorGrain(effective_grain.to_owned())) +} diff --git a/crates/sqlbuild-core/src/plan_display/main/format_no_input_rows.rs b/crates/sqlbuild-core/src/plan_display/main/format_no_input_rows.rs new file mode 100644 index 0000000000..64d582f464 --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/format_no_input_rows.rs @@ -0,0 +1,7 @@ +//! The note for a cursor window whose inputs have no rows. + +/// `no input rows ()`. +#[must_use] +pub fn format_no_input_rows(input_names: &[String]) -> String { + format!("no input rows ({})", input_names.join(", ")) +} diff --git a/crates/sqlbuild-core/src/plan_display/main/format_waiting_on_empty_inputs.rs b/crates/sqlbuild-core/src/plan_display/main/format_waiting_on_empty_inputs.rs new file mode 100644 index 0000000000..6bef827272 --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/format_waiting_on_empty_inputs.rs @@ -0,0 +1,15 @@ +//! The note for a cursor window waiting on empty inputs while others have rows. + +/// `waiting on empty input(s) ; other inputs have new rows`. +#[must_use] +pub fn format_waiting_on_empty_inputs(input_names: &[String]) -> String { + let noun: &str = if input_names.len() == 1 { + "input" + } else { + "inputs" + }; + format!( + "waiting on empty {noun} {}; other inputs have new rows", + input_names.join(", ") + ) +} diff --git a/crates/sqlbuild-core/src/plan_display/main/mod.rs b/crates/sqlbuild-core/src/plan_display/main/mod.rs new file mode 100644 index 0000000000..b2035bea98 --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/mod.rs @@ -0,0 +1,7 @@ +pub mod cursor_bound_display; +pub mod effective_microbatch_batch_size; +pub mod format_no_input_rows; +pub mod format_waiting_on_empty_inputs; +pub mod model_execution_annotation; +pub mod model_materialization_label; +pub mod model_resource_type; diff --git a/crates/sqlbuild-core/src/plan_display/main/model_execution_annotation.rs b/crates/sqlbuild-core/src/plan_display/main/model_execution_annotation.rs new file mode 100644 index 0000000000..d5ce1ee4d7 --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/model_execution_annotation.rs @@ -0,0 +1,22 @@ +//! The annotation an execution progress row shows after the model name. + +use crate::plan_display::_helpers::labels::snapshot_label; +use crate::plan_display::constants::{INCREMENTAL, INCREMENTAL_ACTIONS, SNAPSHOT}; +use crate::plan_display::models::ModelLabelFacts; + +/// The snapshot details, or the incremental strategy of an incremental model. +#[must_use] +pub fn model_execution_annotation(entry: Option<&ModelLabelFacts>) -> String { + let Some(entry) = entry else { + return String::new(); + }; + if entry.materialization_type == SNAPSHOT { + return snapshot_label(entry, false); + } + let incremental: bool = INCREMENTAL_ACTIONS.contains(&entry.action.as_str()) + || entry.materialization_type == INCREMENTAL; + if !incremental { + return String::new(); + } + entry.incremental_strategy.clone().unwrap_or_default() +} diff --git a/crates/sqlbuild-core/src/plan_display/main/model_materialization_label.rs b/crates/sqlbuild-core/src/plan_display/main/model_materialization_label.rs new file mode 100644 index 0000000000..3b2484b0da --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/model_materialization_label.rs @@ -0,0 +1,43 @@ +//! The materialization label of a model in plan summaries. + +use crate::plan_display::_helpers::labels::snapshot_label; +use crate::plan_display::constants::{CUSTOM, INCREMENTAL, MICROBATCH_MODE, SNAPSHOT, TABLE, VIEW}; +use crate::plan_display::models::ModelLabelFacts; + +fn incremental_label(entry: &ModelLabelFacts) -> String { + let strategy: &str = entry + .incremental_strategy + .as_deref() + .filter(|strategy| !strategy.is_empty()) + .unwrap_or(INCREMENTAL); + let mut parts: Vec<&str> = Vec::new(); + if let Some(cursor_type) = &entry.cursor_type { + parts.push(cursor_type); + } + if entry.incremental_mode.as_deref() == Some(MICROBATCH_MODE) { + parts.push(MICROBATCH_MODE); + } + if parts.is_empty() { + return strategy.to_owned(); + } + format!("{strategy} ({})", parts.join(", ")) +} + +/// `view`, `table`, the incremental strategy with its cursor details, the snapshot details or ` (custom)`. +#[must_use] +pub fn model_materialization_label(entry: &ModelLabelFacts) -> String { + match entry.materialization_type.as_str() { + VIEW | TABLE => entry.materialization_type.clone(), + INCREMENTAL => incremental_label(entry), + SNAPSHOT => snapshot_label(entry, true), + CUSTOM => format!( + "{} (custom)", + entry + .custom_materialization_name + .as_deref() + .filter(|name| !name.is_empty()) + .unwrap_or(CUSTOM) + ), + other => other.to_owned(), + } +} diff --git a/crates/sqlbuild-core/src/plan_display/main/model_resource_type.rs b/crates/sqlbuild-core/src/plan_display/main/model_resource_type.rs new file mode 100644 index 0000000000..22b730189d --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/model_resource_type.rs @@ -0,0 +1,16 @@ +//! The left-column resource type of an execution progress row. + +use crate::plan_display::constants::{CUSTOM, SNAPSHOT, TABLE, VIEW}; +use crate::plan_display::models::ModelLabelFacts; + +/// `view`, `custom` or `snapshot` for those materializations, otherwise `table`. +#[must_use] +pub fn model_resource_type(entry: Option<&ModelLabelFacts>) -> &'static str { + let Some(entry) = entry else { + return TABLE; + }; + [VIEW, CUSTOM, SNAPSHOT] + .into_iter() + .find(|kind| entry.materialization_type == *kind) + .unwrap_or(TABLE) +} diff --git a/crates/sqlbuild-core/src/plan_display/mod.rs b/crates/sqlbuild-core/src/plan_display/mod.rs new file mode 100644 index 0000000000..bdd383ca79 --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/mod.rs @@ -0,0 +1,10 @@ +//! Plan and progress display fragments: materialization labels, cursor bound display, empty-input notes and batch sizes. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/plan_display/models.rs b/crates/sqlbuild-core/src/plan_display/models.rs new file mode 100644 index 0000000000..e1072e7392 --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/models.rs @@ -0,0 +1,25 @@ +//! A planned model as its materialization labels read it. + +/// The plan-entry fields materialization labels read. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModelLabelFacts { + pub materialization_type: String, + pub action: String, + pub custom_materialization_name: Option, + pub incremental_strategy: Option, + pub incremental_mode: Option, + pub cursor_type: Option, + pub snapshot_strategy: Option, + /// The snapshot records an `observed_at` column. + pub historical: bool, + pub historical_input: Option, +} + +/// A cursor bound value parsed as an ISO date or datetime. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ParsedBound { + /// The value's calendar date as `YYYY-MM-DD`. + pub date: String, + /// Hour, minute, second and microsecond are all zero. + pub midnight: bool, +} diff --git a/crates/sqlbuild-core/src/plan_display/tests.rs b/crates/sqlbuild-core/src/plan_display/tests.rs new file mode 100644 index 0000000000..28a494f522 --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_plan_display.rs"] +mod plan_display; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/plan_display/tests/helpers.rs b/crates/sqlbuild-core/src/plan_display/tests/helpers.rs new file mode 100644 index 0000000000..047946bc2d --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/tests/helpers.rs @@ -0,0 +1,10 @@ +use crate::plan_display::models::ModelLabelFacts; + +/// A planned model with only its materialization and action. +pub(super) fn facts(materialization_type: &str, action: &str) -> ModelLabelFacts { + ModelLabelFacts { + materialization_type: materialization_type.to_owned(), + action: action.to_owned(), + ..ModelLabelFacts::default() + } +} diff --git a/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs b/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs new file mode 100644 index 0000000000..0e8ed49570 --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs @@ -0,0 +1,209 @@ +use crate::plan_display::main::cursor_bound_display::cursor_bound_display; +use crate::plan_display::main::effective_microbatch_batch_size::effective_microbatch_batch_size; +use crate::plan_display::main::format_no_input_rows::format_no_input_rows; +use crate::plan_display::main::format_waiting_on_empty_inputs::format_waiting_on_empty_inputs; +use crate::plan_display::main::model_execution_annotation::model_execution_annotation; +use crate::plan_display::main::model_materialization_label::model_materialization_label; +use crate::plan_display::main::model_resource_type::model_resource_type; +use crate::plan_display::models::{ModelLabelFacts, ParsedBound}; +use crate::plan_display::tests::helpers::facts; +use crate::plan_display::tests::test_types::{ + BatchSizeTestCase, BoundTestCase, LabelTestCase, NamesTestCase, +}; + +#[test] +fn given_planned_model_when_labelling_then_matches_plan_and_progress_text() { + let test_cases = [ + LabelTestCase { + description: "a view", + entry: facts("view", "create_view"), + expected_label: "view", + expected_resource_type: "view", + expected_annotation: "", + }, + LabelTestCase { + description: "a microbatch incremental with a cursor type", + entry: ModelLabelFacts { + incremental_strategy: Some("delete_insert".to_owned()), + incremental_mode: Some("microbatch".to_owned()), + cursor_type: Some("timestamp".to_owned()), + ..facts("incremental", "incremental_delete_insert") + }, + expected_label: "delete_insert (timestamp, microbatch)", + expected_resource_type: "table", + expected_annotation: "delete_insert", + }, + LabelTestCase { + description: "an incremental without a strategy", + entry: ModelLabelFacts { + incremental_strategy: Some(String::new()), + ..facts("incremental", "full_build") + }, + expected_label: "incremental", + expected_resource_type: "table", + expected_annotation: "", + }, + LabelTestCase { + description: "a historical snapshot", + entry: ModelLabelFacts { + snapshot_strategy: Some("timestamp".to_owned()), + historical: true, + ..facts("snapshot", "snapshot") + }, + expected_label: "snapshot (timestamp, historical snapshot)", + expected_resource_type: "snapshot", + expected_annotation: "timestamp, historical snapshot", + }, + LabelTestCase { + description: "a bare snapshot", + entry: facts("snapshot", "snapshot"), + expected_label: "snapshot", + expected_resource_type: "snapshot", + expected_annotation: "", + }, + LabelTestCase { + description: "a custom materialization", + entry: ModelLabelFacts { + custom_materialization_name: Some("partition_tracked".to_owned()), + ..facts("custom", "custom") + }, + expected_label: "partition_tracked (custom)", + expected_resource_type: "custom", + expected_annotation: "", + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + model_materialization_label(&test_case.entry), + model_resource_type(Some(&test_case.entry)), + model_execution_annotation(Some(&test_case.entry)), + ), + ( + test_case.expected_label.to_owned(), + test_case.expected_resource_type, + test_case.expected_annotation.to_owned(), + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_cursor_bound_when_displaying_then_collapses_whole_day_midnights() { + let midnight: ParsedBound = ParsedBound { + date: "2026-01-02".to_owned(), + midnight: true, + }; + let test_cases = [ + BoundTestCase { + description: "a day-grain midnight collapses to its date", + value: "2026-01-02T00:00:00", + typing: (Some("timestamp"), Some("day")), + parsed: Some(midnight.clone()), + expected_text: "2026-01-02", + }, + BoundTestCase { + description: "an hour grain keeps the time", + value: "2026-01-02T00:00:00", + typing: (Some("timestamp"), Some("hour")), + parsed: Some(midnight.clone()), + expected_text: "2026-01-02T00:00:00", + }, + BoundTestCase { + description: "an integer cursor is shown as is", + value: "42", + typing: (Some("integer"), Some("day")), + parsed: Some(midnight), + expected_text: "42", + }, + BoundTestCase { + description: "an unparsed value is shown as is", + value: "later", + typing: (None, Some("day")), + parsed: None, + expected_text: "later", + }, + ]; + + for test_case in test_cases { + assert_eq!( + cursor_bound_display(test_case.value, test_case.typing, test_case.parsed.as_ref()), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_empty_inputs_when_formatting_then_names_them() { + let test_cases = [ + NamesTestCase { + description: "one input", + names: vec!["orders".to_owned()], + expected_no_rows: "no input rows (orders)", + expected_waiting: "waiting on empty input orders; other inputs have new rows", + }, + NamesTestCase { + description: "two inputs", + names: vec!["orders".to_owned(), "returns".to_owned()], + expected_no_rows: "no input rows (orders, returns)", + expected_waiting: "waiting on empty inputs orders, returns; other inputs have new rows", + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + format_no_input_rows(&test_case.names), + format_waiting_on_empty_inputs(&test_case.names) + ), + ( + test_case.expected_no_rows.to_owned(), + test_case.expected_waiting.to_owned() + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_batch_size_when_resolving_effective_size_then_uses_grain_unit() { + let test_cases = [ + BatchSizeTestCase { + description: "a declared size is kept", + batch_size: "2d", + grain: "hour", + expected_size: Ok("2d"), + }, + BatchSizeTestCase { + description: "effective follows the grain", + batch_size: "effective", + grain: "month", + expected_size: Ok("1mo"), + }, + BatchSizeTestCase { + description: "an unknown grain is an error", + batch_size: "effective", + grain: "week", + expected_size: Err("week"), + }, + ]; + + for test_case in test_cases { + assert_eq!( + effective_microbatch_batch_size(test_case.batch_size, test_case.grain) + .map_err(|error| error.0), + test_case + .expected_size + .map(str::to_owned) + .map_err(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/plan_display/tests/test_types.rs b/crates/sqlbuild-core/src/plan_display/tests/test_types.rs new file mode 100644 index 0000000000..08cbf72a61 --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/tests/test_types.rs @@ -0,0 +1,31 @@ +use crate::plan_display::models::{ModelLabelFacts, ParsedBound}; + +pub(super) struct LabelTestCase { + pub(super) description: &'static str, + pub(super) entry: ModelLabelFacts, + pub(super) expected_label: &'static str, + pub(super) expected_resource_type: &'static str, + pub(super) expected_annotation: &'static str, +} + +pub(super) struct BoundTestCase { + pub(super) description: &'static str, + pub(super) value: &'static str, + pub(super) typing: (Option<&'static str>, Option<&'static str>), + pub(super) parsed: Option, + pub(super) expected_text: &'static str, +} + +pub(super) struct NamesTestCase { + pub(super) description: &'static str, + pub(super) names: Vec, + pub(super) expected_no_rows: &'static str, + pub(super) expected_waiting: &'static str, +} + +pub(super) struct BatchSizeTestCase { + pub(super) description: &'static str, + pub(super) batch_size: &'static str, + pub(super) grain: &'static str, + pub(super) expected_size: Result<&'static str, &'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs index 259fa34a09..c6582585cb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs @@ -194,7 +194,7 @@ fn parse<'py>( /// Discovery values that may be dates or timestamps, or `None` when they do not parse. #[pyfunction(name = "cursor_try_parse")] #[pyo3(signature = (*, raw, cursor_type))] -fn try_parse<'py>( +pub(crate) fn try_parse<'py>( raw: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, ) -> PyResult>> { @@ -363,7 +363,7 @@ fn next_value<'py>( /// An operator-declared inclusive bound advanced by one cursor unit. #[pyfunction(name = "cursor_inclusive_to_exclusive")] #[pyo3(signature = (*, value, grain))] -fn to_exclusive<'py>( +pub(crate) fn to_exclusive<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, ) -> PyResult> { @@ -373,7 +373,7 @@ fn to_exclusive<'py>( /// An exclusive bound moved back by one cursor unit. #[pyfunction(name = "cursor_exclusive_to_inclusive")] #[pyo3(signature = (*, value, grain))] -fn to_inclusive<'py>( +pub(crate) fn to_inclusive<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs index 40a9d2d458..f34fba5aab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs @@ -2,15 +2,14 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::plan_output::display::{ + aligned_timestamp_bounds, resolve_effective_microbatch_batch_size, +}; use crate::bindings::_helpers::plan_output::values::{optional_text, text}; const BOUND_TYPES_MODULE: &str = "sqlbuild.cursor_algebra.types"; const CURSOR_MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; -const ALIGNED_BOUNDS_MODULE: &str = - "sqlbuild.compiler.planner.main.execution.aligned_timestamp_bounds"; -const BATCH_SIZE_MODULE: &str = - "sqlbuild.compiler.planner.main.execution.effective_microbatch_batch_size"; const MICROBATCH_MODE: &str = "microbatch"; const TIMESTAMP_CURSOR: &str = "timestamp"; const INTEGER_CURSOR: &str = "integer"; @@ -99,11 +98,8 @@ fn count_batches( if !temporal(&start)? || !temporal(&end)? { return Ok(None); } - let (mut current, end): (Bound<'_, PyAny>, Bound<'_, PyAny>) = call_with( - &imported(py, ALIGNED_BOUNDS_MODULE, "aligned_timestamp_bounds")?, - &[("start", start), ("end", end)], - )? - .extract()?; + let (mut current, end): (Bound<'_, PyAny>, Bound<'_, PyAny>) = + aligned_timestamp_bounds(&start, &end)?.extract()?; let minimum: Bound<'_, PyAny> = py.import("builtins")?.getattr("min")?; let mut count: i64 = 0; while current.lt(&end)? { @@ -167,23 +163,9 @@ pub(crate) fn cursor_details<'py>( let effective_grain: Option = effective_grain(entry)?; let declared_batch_size: Option = optional_text(&entry.getattr("batch_size")?)?; let effective_batch_size: Option = match (&declared_batch_size, &effective_grain) { - (Some(batch_size), Some(grain)) => Some(text(&call_with( - &imported( - py, - BATCH_SIZE_MODULE, - "resolve_effective_microbatch_batch_size", - )?, - &[ - ( - "batch_size", - pyo3::types::PyString::new(py, batch_size).into_any(), - ), - ( - "effective_grain", - pyo3::types::PyString::new(py, grain).into_any(), - ), - ], - )?)?), + (Some(batch_size), Some(grain)) => Some(resolve_effective_microbatch_batch_size( + py, batch_size, grain, + )?), (batch_size, _) => batch_size.clone(), }; let cursor_type: Option = optional_text(&entry.getattr("cursor_type")?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs new file mode 100644 index 0000000000..52abfd9a36 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs @@ -0,0 +1,284 @@ +//! Plan and progress display fragments over planner entries and cursor values. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::plan_display::constants::{ + DEFAULT_CURSOR_GRAIN, INTEGER_CURSOR, TIMESTAMP_CURSOR, +}; +use sqlbuild_core::plan_display::main::cursor_bound_display::cursor_bound_display as bound_display; +use sqlbuild_core::plan_display::main::effective_microbatch_batch_size::effective_microbatch_batch_size; +use sqlbuild_core::plan_display::main::format_no_input_rows::format_no_input_rows as no_input_rows; +use sqlbuild_core::plan_display::main::format_waiting_on_empty_inputs::format_waiting_on_empty_inputs as waiting_on_empty_inputs; +use sqlbuild_core::plan_display::main::model_execution_annotation::model_execution_annotation as execution_annotation; +use sqlbuild_core::plan_display::main::model_materialization_label::model_materialization_label as materialization_label; +use sqlbuild_core::plan_display::main::model_resource_type::model_resource_type as resource_type; +use sqlbuild_core::plan_display::models::{ModelLabelFacts, ParsedBound}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cursor::scalars::{ + render_value, to_exclusive, to_inclusive, try_parse, +}; +use crate::bindings::_helpers::plan_output::values::{optional_text, text}; + +const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; +const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +const CURSOR_MODELS: &str = "sqlbuild.cursor_algebra.models"; +const DATETIME_MODULE: &str = "datetime"; + +fn label_facts(entry: &Bound<'_, PyAny>) -> PyResult { + let field = |name: &str| -> PyResult> { optional_text(&entry.getattr(name)?) }; + Ok(ModelLabelFacts { + materialization_type: text(&entry.getattr("materialization_type")?)?, + action: text(&entry.getattr("action")?)?, + custom_materialization_name: field("custom_materialization_name")?, + incremental_strategy: field("incremental_strategy")?, + incremental_mode: field("incremental_mode")?, + cursor_type: field("cursor_type")?, + snapshot_strategy: field("snapshot_strategy")?, + historical: !entry.getattr("observed_at_column")?.is_none(), + historical_input: field("historical_input")?, + }) +} + +fn optional_facts(entry: &Bound<'_, PyAny>) -> PyResult> { + if entry.is_none() { + return Ok(None); + } + label_facts(entry).map(Some) +} + +/// The cursor type, defaulting to timestamp, and its grain object (`None` for integers). +fn cursor_typing<'py>( + cursor_type: &Bound<'py, PyAny>, + cursor_grain: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = cursor_type.py(); + let effective: Bound<'py, PyAny> = if cursor_type.is_truthy()? { + cursor_type.clone() + } else { + PyString::new(py, TIMESTAMP_CURSOR).into_any() + }; + if effective.eq(INTEGER_CURSOR)? { + return Ok((effective, py.None().into_bound(py))); + } + let grain: Bound<'py, PyAny> = if cursor_grain.is_truthy()? { + cursor_grain.clone() + } else { + PyString::new(py, DEFAULT_CURSOR_GRAIN).into_any() + }; + let grain: Bound<'py, PyAny> = imported(py, PLANNER_TYPES, "CursorGrain")?.call1((grain,))?; + Ok((effective, grain)) +} + +/// The last cursor value included by an exclusive end bound. +#[pyfunction] +#[pyo3(signature = (*, end, cursor_type, cursor_grain))] +pub(crate) fn inclusive_cursor_end( + end: &Bound<'_, PyAny>, + cursor_type: &Bound<'_, PyAny>, + cursor_grain: &Bound<'_, PyAny>, +) -> PyResult { + let (effective, grain) = cursor_typing(cursor_type, cursor_grain)?; + let Some(parsed) = try_parse(end, &effective)? else { + return end.extract(); + }; + render_value(&to_inclusive(&parsed, &grain)?) +} + +/// A fully bounded inclusive operator override as an executable interval. +#[pyfunction] +#[pyo3(signature = (*, start_cursor_override, end_cursor_override, cursor_type, cursor_grain))] +fn resolve_bounded_cursor_override<'py>( + start_cursor_override: &Bound<'py, PyAny>, + end_cursor_override: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, + cursor_grain: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = start_cursor_override.py(); + if start_cursor_override.is_none() || end_cursor_override.is_none() { + return Ok(None); + } + let (effective, grain) = cursor_typing(cursor_type, cursor_grain)?; + let (Some(start), Some(end)) = ( + try_parse(start_cursor_override, &effective)?, + try_parse(end_cursor_override, &effective)?, + ) else { + return Ok(None); + }; + call_with( + &imported(py, PLANNER_MODELS, "CursorBounds")?, + &[("start", start), ("end", to_exclusive(&end, &grain)?)], + ) + .map(Some) +} + +fn parsed_bound(value: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = value.py(); + let parsed: Bound<'_, PyAny> = + match imported(py, DATETIME_MODULE, "datetime")?.call_method1("fromisoformat", (value,)) { + Ok(parsed) => parsed, + Err(error) if error.is_instance_of::(py) => return Ok(None), + Err(error) => return Err(error), + }; + let mut midnight: bool = true; + for part in ["hour", "minute", "second", "microsecond"] { + midnight = midnight && parsed.getattr(part)?.extract::()? == 0; + } + Ok(Some(ParsedBound { + date: parsed + .call_method0("date")? + .call_method0("isoformat")? + .extract()?, + midnight, + })) +} + +/// A cursor bound for display, collapsing a whole-day midnight timestamp to a date. +#[pyfunction] +#[pyo3(signature = (*, value, cursor_type, cursor_grain))] +pub(crate) fn cursor_bound_display( + value: &Bound<'_, PyAny>, + cursor_type: &Bound<'_, PyAny>, + cursor_grain: &Bound<'_, PyAny>, +) -> PyResult { + let cursor_type: Option = optional_text(cursor_type)?; + let raw: String = value.extract()?; + let parsed: Option = if cursor_type.as_deref() == Some(INTEGER_CURSOR) { + None + } else { + parsed_bound(value)? + }; + Ok(bound_display( + &raw, + ( + cursor_type.as_deref(), + optional_text(cursor_grain)?.as_deref(), + ), + parsed.as_ref(), + )) +} + +/// The full materialization label used in plan summaries. +#[pyfunction] +pub(crate) fn model_materialization_label(entry: &Bound<'_, PyAny>) -> PyResult { + Ok(materialization_label(&label_facts(entry)?)) +} + +/// The left-column resource type used in execution progress rows. +#[pyfunction] +fn model_resource_type(entry: &Bound<'_, PyAny>) -> PyResult<&'static str> { + Ok(resource_type(optional_facts(entry)?.as_ref())) +} + +/// The model annotation used in execution progress rows. +#[pyfunction] +fn model_execution_annotation(entry: &Bound<'_, PyAny>) -> PyResult { + Ok(execution_annotation(optional_facts(entry)?.as_ref())) +} + +/// The left-column resource type for active progress rows. +#[pyfunction] +fn materialization_type_display<'py>( + resource_kind: &Bound<'py, PyAny>, +) -> PyResult> { + resource_kind.getattr("value") +} + +fn names(input_names: &Bound<'_, PyAny>) -> PyResult> { + input_names + .try_iter()? + .map(|name| name?.extract()) + .collect() +} + +/// Explain that a cursor window is empty because its inputs have no rows. +#[pyfunction] +pub(crate) fn format_no_input_rows(input_names: &Bound<'_, PyAny>) -> PyResult { + Ok(no_input_rows(&names(input_names)?)) +} + +/// Explain that a cursor window waits on empty inputs while other inputs have rows. +#[pyfunction] +pub(crate) fn format_waiting_on_empty_inputs(input_names: &Bound<'_, PyAny>) -> PyResult { + Ok(waiting_on_empty_inputs(&names(input_names)?)) +} + +/// The explicit effective batch-size token resolved against the replay grain. +#[pyfunction] +#[pyo3(signature = (*, batch_size, effective_grain))] +pub(crate) fn resolve_effective_microbatch_batch_size( + py: Python<'_>, + batch_size: &str, + effective_grain: &str, +) -> Result { + effective_microbatch_batch_size(batch_size, effective_grain).map_err(|error| { + imported(py, PLANNER_TYPES, "CursorGrain") + .and_then(|grain| grain.call1((error.0,))) + .err() + .unwrap_or_else(|| PyValueError::new_err("unknown cursor grain")) + }) +} + +fn as_datetime<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + let inner: Bound<'py, PyAny> = value.getattr("value")?; + if value.is_instance(&imported(py, CURSOR_MODELS, "TimestampValue")?)? { + return Ok(inner); + } + let datetime: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; + let midnight: Bound<'py, PyAny> = datetime.getattr("min")?.call_method0("time")?; + datetime.call_method1("combine", (inner, midnight)) +} + +fn presume_utc<'py>(value: Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + if !value.getattr("tzinfo")?.is_none() { + return Ok(value); + } + let options: Bound<'py, PyDict> = PyDict::new(py); + pyo3::types::PyDictMethods::set_item( + &options, + "tzinfo", + imported(py, DATETIME_MODULE, "UTC")?, + )?; + value.call_method("replace", (), Some(&options)) +} + +/// Comparable bounds, presuming a naive side UTC when the other is aware. +#[pyfunction] +#[pyo3(signature = (*, start, end))] +pub(crate) fn aligned_timestamp_bounds<'py>( + start: &Bound<'py, PyAny>, + end: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = start.py(); + let mut start_at: Bound<'py, PyAny> = as_datetime(start)?; + let mut end_at: Bound<'py, PyAny> = as_datetime(end)?; + if start_at.getattr("tzinfo")?.is_none() != end_at.getattr("tzinfo")?.is_none() { + start_at = presume_utc(start_at)?; + end_at = presume_utc(end_at)?; + } + PyTuple::new(py, [start_at, end_at]) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(inclusive_cursor_end, module)?)?; + module.add_function(wrap_pyfunction!(resolve_bounded_cursor_override, module)?)?; + module.add_function(wrap_pyfunction!(cursor_bound_display, module)?)?; + module.add_function(wrap_pyfunction!(model_materialization_label, module)?)?; + module.add_function(wrap_pyfunction!(model_resource_type, module)?)?; + module.add_function(wrap_pyfunction!(model_execution_annotation, module)?)?; + module.add_function(wrap_pyfunction!(materialization_type_display, module)?)?; + module.add_function(wrap_pyfunction!(format_no_input_rows, module)?)?; + module.add_function(wrap_pyfunction!(format_waiting_on_empty_inputs, module)?)?; + module.add_function(wrap_pyfunction!( + resolve_effective_microbatch_batch_size, + module + )?)?; + module.add_function(wrap_pyfunction!(aligned_timestamp_bounds, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs index 11ea8bead3..fd834ed5b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod cursor_details; +pub(crate) mod display; pub(crate) mod plan_json; pub(crate) mod plan_text; pub(crate) mod python_identity; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs index 308ae74ccb..11392a2f4e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs @@ -18,20 +18,16 @@ use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::plan_output::cursor_details::{ CursorDetails, DEFERRED, RESOLVED, cursor_details, runtime_inputs_clause, }; +use crate::bindings::_helpers::plan_output::display::{ + cursor_bound_display, format_no_input_rows, format_waiting_on_empty_inputs, + inclusive_cursor_end, model_materialization_label, +}; use crate::bindings::_helpers::plan_output::python_identity::{ definition_source_text, dependency_source_text, }; use crate::bindings::_helpers::plan_output::values::{items, optional_text, text, texts}; const QUERY_DIFF_MODULE: &str = "sqlbuild.compiler.planner.main.changes._query_diff"; -const NO_INPUT_ROWS_MODULE: &str = - "sqlbuild.compiler.planner.main.cursor_window.no_input_rows_display"; -const WAITING_MODULE: &str = - "sqlbuild.compiler.planner.main.cursor_window.waiting_on_empty_inputs_display"; -const BOUND_DISPLAY_MODULE: &str = "sqlbuild.compiler.planner.main.execution.cursor_bound_display"; -const INCLUSIVE_END_MODULE: &str = "sqlbuild.compiler.planner.main.execution.inclusive_cursor_end"; -const MATERIALIZATION_LABEL_MODULE: &str = - "sqlbuild.compiler.planner.main.execution._model_materialization_label"; const PRE_BUILD_MODULE: &str = "sqlbuild.compiler.planner.main.pre_build.pre_build_work_display"; const DISPLAY_MODELS_MODULE: &str = "sqlbuild.presentation.models"; const SKIP_ACTION: &str = "skip"; @@ -70,45 +66,27 @@ fn token<'py>(bound: Bound<'py, PyAny>) -> PyResult> { /// `start → inclusive end` of resolved bounds, in the model's cursor display. fn cursor_range(entry: &Bound<'_, PyAny>, bounds: &Bound<'_, PyAny>) -> PyResult { - let py: Python<'_> = entry.py(); - let typing: [(&str, Bound<'_, PyAny>); 2] = [ - ("cursor_type", entry.getattr("cursor_type")?), - ("cursor_grain", entry.getattr("cursor_grain")?), - ]; - let start: String = call_with( - &imported(py, BOUND_DISPLAY_MODULE, "cursor_bound_display")?, - &[ - ("value", token(bounds.getattr("start")?)?), - typing[0].clone(), - typing[1].clone(), - ], - )? - .extract()?; - let end: String = call_with( - &imported(py, INCLUSIVE_END_MODULE, "inclusive_cursor_end")?, - &[ - ("end", token(bounds.getattr("end")?)?), - typing[0].clone(), - typing[1].clone(), - ], - )? - .extract()?; + let (cursor_type, cursor_grain) = ( + entry.getattr("cursor_type")?, + entry.getattr("cursor_grain")?, + ); + let start: String = cursor_bound_display( + &token(bounds.getattr("start")?)?, + &cursor_type, + &cursor_grain, + )?; + let end: String = + inclusive_cursor_end(&token(bounds.getattr("end")?)?, &cursor_type, &cursor_grain)?; Ok(format!("{start} \u{2192} {end}")) } fn cursor_text(entry: &Bound<'_, PyAny>, details: CursorDetails<'_>) -> PyResult { - let py: Python<'_> = entry.py(); let empty_inputs_window: Option = if details.empty_inputs.is_truthy()? { - let (module, function) = if details.waiting_on_empty_inputs { - (WAITING_MODULE, "format_waiting_on_empty_inputs") + Some(if details.waiting_on_empty_inputs { + format_waiting_on_empty_inputs(&details.empty_inputs)? } else { - (NO_INPUT_ROWS_MODULE, "format_no_input_rows") - }; - Some( - imported(py, module, function)? - .call1((&details.empty_inputs,))? - .extract()?, - ) + format_no_input_rows(&details.empty_inputs)? + }) } else { None }; @@ -185,7 +163,6 @@ fn config_diff(entry: &Bound<'_, PyAny>, reason: &str) -> PyResult) -> PyResult { - let py: Python<'_> = entry.py(); let action: String = text(&entry.getattr("action")?.getattr("value")?)?; let reason: String = text(&entry.getattr("reason")?.getattr("value")?)?; let name: String = text(&entry.getattr("name")?)?; @@ -226,13 +203,7 @@ fn model_text(entry: &Bound<'_, PyAny>) -> PyResult { )?) }; Ok(ModelPlanText { - materialization_label: imported( - py, - MATERIALIZATION_LABEL_MODULE, - "model_materialization_label", - )? - .call1((entry,))? - .extract()?, + materialization_label: model_materialization_label(entry)?, custom_materialization_name: optional_text(&entry.getattr("custom_materialization_name")?)?, backfill_action: text(&backfill.getattr("action")?.getattr("value")?)?, backfill_duration: optional_text(&backfill.getattr("duration")?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs index ce31fcb06b..cd0fab2819 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::plan_output::{plan_json, plan_text, safety}; +use crate::bindings::_helpers::plan_output::{display, plan_json, plan_text, safety}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + display::register(module)?; plan_json::register(module)?; plan_text::register(module)?; safety::register(module) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index ffb37dbfac..55759b3dd8 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -177,9 +177,11 @@ from sqlbuild.compiler.pipeline.models import ( ) from sqlbuild.compiler.planner.models import ( AuditPlanEntry, + CursorBounds, DeferralInputs, FutureCursorSafetyEvidence, MaximumStartSafetyEvidence, + ModelPlanEntry, PlanOutput, SqlTestPlanEntry, ) @@ -238,7 +240,12 @@ from sqlbuild.compiler.sql_test_glue.models import ( ) from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestPlanRow from sqlbuild.cost.models import CostRunRecord -from sqlbuild.cursor_algebra.models import AlignedInterval, Duration +from sqlbuild.cursor_algebra.models import ( + AlignedInterval, + DateValue, + Duration, + TimestampValue, +) from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult @@ -272,6 +279,7 @@ from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.rule_engine.models import Finding, Rule, RulesConfig, RulesResult from sqlbuild.rule_engine.types import RuleCheck from sqlbuild.runtime.contracts.models import ConnectionHooks +from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.runtime.observability.models import LifecycleEvent from sqlbuild.spec.contracts.models import ( DefaultsConfig, @@ -3416,3 +3424,24 @@ def parse_header_values( def model_output_column_locations( *, contents: str, relative_path: object, extract_implicit_alias_columns: bool = True ) -> dict[str, Any]: ... +def inclusive_cursor_end(*, end: str, cursor_type: str | None, cursor_grain: str | None) -> str: ... +def resolve_bounded_cursor_override( + *, + start_cursor_override: str | None, + end_cursor_override: str | None, + cursor_type: str | None, + cursor_grain: str | None, +) -> CursorBounds | None: ... +def cursor_bound_display( + *, value: str, cursor_type: str | None, cursor_grain: str | None +) -> str: ... +def model_materialization_label(entry: ModelPlanEntry) -> str: ... +def model_resource_type(entry: ModelPlanEntry | None) -> str: ... +def model_execution_annotation(entry: ModelPlanEntry | None) -> str: ... +def materialization_type_display(resource_kind: ExecutionResourceKind) -> str: ... +def format_no_input_rows(input_names: tuple[str, ...]) -> str: ... +def format_waiting_on_empty_inputs(input_names: tuple[str, ...]) -> str: ... +def resolve_effective_microbatch_batch_size(*, batch_size: str, effective_grain: str) -> str: ... +def aligned_timestamp_bounds( + *, start: DateValue | TimestampValue, end: DateValue | TimestampValue +) -> tuple[datetime, datetime]: ... diff --git a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py index 8beb7512a2..48a1ea6a9e 100644 --- a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py +++ b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py @@ -13,31 +13,25 @@ ExecutionEventWriter, NativeProgressProjector, current_native_progress_projector, + cursor_bound_display, format_completion_line, format_expectation_detail, format_expectation_name, + format_no_input_rows, format_parameterized_test_label, format_status_cell, format_summary_footer, + format_waiting_on_empty_inputs, + inclusive_cursor_end, + materialization_type_display, + model_execution_annotation, + model_resource_type, ) from sqlbuild.adapter.contract.models import LifeCycleEvent from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.cli.progress.models import AuditDisplayEntry, ExecutionCounts from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditOutcome, AuditRunScope from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.main.cursor_window.no_input_rows_display import format_no_input_rows -from sqlbuild.compiler.planner.main.cursor_window.waiting_on_empty_inputs_display import ( - format_waiting_on_empty_inputs, -) -from sqlbuild.compiler.planner.main.execution.cursor_bound_display import cursor_bound_display -from sqlbuild.compiler.planner.main.execution.inclusive_cursor_end import inclusive_cursor_end -from sqlbuild.compiler.planner.main.execution.materialization_type_display import ( - materialization_type_display, -) -from sqlbuild.compiler.planner.main.execution.model_execution_annotation import ( - model_execution_annotation, -) -from sqlbuild.compiler.planner.main.execution.model_resource_type import model_resource_type from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput from sqlbuild.compiler.python_nodes.types import PythonNodeStatus from sqlbuild.cost.models import StatementExecutionTelemetry diff --git a/src/sqlbuild/compiler/planner/_helpers/output/inclusive_cursor_end.py b/src/sqlbuild/compiler/planner/_helpers/output/inclusive_cursor_end.py deleted file mode 100644 index c01cc73259..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/inclusive_cursor_end.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Single source of truth for converting between inclusive and exclusive cursor end bounds.""" - -from __future__ import annotations - -from datetime import datetime - -from sqlbuild._native import cursor_exclusive_to_inclusive as exclusive_to_inclusive -from sqlbuild._native import cursor_inclusive_to_exclusive as inclusive_to_exclusive -from sqlbuild._native import cursor_render as render -from sqlbuild._native import cursor_try_parse as try_parse -from sqlbuild.compiler.planner.constants import WHOLE_DAY_CURSOR_GRAINS -from sqlbuild.compiler.planner.models import CursorBounds -from sqlbuild.compiler.planner.types import CursorGrain, CursorType -from sqlbuild.cursor_algebra.types import CursorScalar - - -def inclusive_cursor_end( - *, - end: str, - cursor_type: str | None, - cursor_grain: str | None, -) -> str: - """Return the last cursor value included by an exclusive end bound.""" - - effective_type: str = cursor_type or CursorType.TIMESTAMP - parsed: CursorScalar | None = try_parse(raw=end, cursor_type=effective_type) - if parsed is None: - return end - grain: CursorGrain | None = ( - None - if effective_type == CursorType.INTEGER - else CursorGrain(cursor_grain or CursorGrain.SECOND) - ) - return render(value=exclusive_to_inclusive(value=parsed, grain=grain)) - - -def cursor_bound_display( - *, - value: str, - cursor_type: str | None, - cursor_grain: str | None, -) -> str: - """Render a cursor bound for display, collapsing a whole-day midnight timestamp to a date.""" - - if cursor_type == CursorType.INTEGER: - return value - try: - parsed: datetime = datetime.fromisoformat(value) - except ValueError: - return value - if cursor_grain in WHOLE_DAY_CURSOR_GRAINS and _has_no_time_component(value=parsed): - return parsed.date().isoformat() - return value - - -def resolve_bounded_cursor_override( - *, - start_cursor_override: str | None, - end_cursor_override: str | None, - cursor_type: str | None, - cursor_grain: str | None, -) -> CursorBounds | None: - """Resolve a fully bounded inclusive operator override to an executable interval.""" - - if start_cursor_override is None or end_cursor_override is None: - return None - effective_type: str = cursor_type or CursorType.TIMESTAMP - start: CursorScalar | None = try_parse(raw=start_cursor_override, cursor_type=effective_type) - inclusive_end: CursorScalar | None = try_parse( - raw=end_cursor_override, cursor_type=effective_type - ) - if start is None or inclusive_end is None: - return None - grain: CursorGrain | None = ( - None - if effective_type == CursorType.INTEGER - else CursorGrain(cursor_grain or CursorGrain.SECOND) - ) - return CursorBounds( - start=start, - end=inclusive_to_exclusive(value=inclusive_end, grain=grain), - ) - - -def _has_no_time_component(*, value: datetime) -> bool: - return (value.hour, value.minute, value.second, value.microsecond) == (0, 0, 0, 0) diff --git a/src/sqlbuild/compiler/planner/_helpers/output/materialization_labels.py b/src/sqlbuild/compiler/planner/_helpers/output/materialization_labels.py deleted file mode 100644 index 4a835d3b70..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/materialization_labels.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Model materialization display label implementations.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.compiler.planner.types import ( - HistoricalInput, - IncrementalMode, - MaterializationType, - PlanAction, -) -from sqlbuild.runtime.contracts.types import ExecutionResourceKind - -_INCREMENTAL_ACTIONS: frozenset[PlanAction] = frozenset( - { - PlanAction.INCREMENTAL_APPEND, - PlanAction.INCREMENTAL_DELETE_INSERT, - PlanAction.INCREMENTAL_MERGE, - } -) - - -def model_materialization_label(entry: ModelPlanEntry) -> str: - if entry.materialization_type == MaterializationType.VIEW: - return MaterializationType.VIEW.value - if entry.materialization_type == MaterializationType.TABLE: - return MaterializationType.TABLE.value - if entry.materialization_type == MaterializationType.INCREMENTAL: - return _incremental_label(entry) - if entry.materialization_type == MaterializationType.SNAPSHOT: - return _snapshot_label(entry=entry, include_prefix=True) - if entry.materialization_type == MaterializationType.CUSTOM: - custom_name: str = entry.custom_materialization_name or MaterializationType.CUSTOM.value - return f"{custom_name} (custom)" - return entry.materialization_type.value - - -def model_resource_type(entry: ModelPlanEntry | None) -> str: - if entry is None: - return MaterializationType.TABLE.value - if entry.materialization_type == MaterializationType.VIEW: - return MaterializationType.VIEW.value - if entry.materialization_type == MaterializationType.CUSTOM: - return MaterializationType.CUSTOM.value - if entry.materialization_type == MaterializationType.SNAPSHOT: - return MaterializationType.SNAPSHOT.value - return MaterializationType.TABLE.value - - -def materialization_type_display(resource_kind: ExecutionResourceKind) -> str: - return resource_kind.value - - -def model_execution_annotation(entry: ModelPlanEntry | None) -> str: - if entry is None: - return "" - if entry.materialization_type == MaterializationType.SNAPSHOT: - return _snapshot_label(entry=entry, include_prefix=False) - is_incremental: bool = ( - entry.action in _INCREMENTAL_ACTIONS - or entry.materialization_type == MaterializationType.INCREMENTAL - ) - if not is_incremental: - return "" - if entry.incremental_strategy: - return entry.incremental_strategy - return "" - - -def _incremental_label(entry: ModelPlanEntry) -> str: - strategy: str = entry.incremental_strategy or MaterializationType.INCREMENTAL.value - parts: list[str] = [] - if entry.cursor_type is not None: - parts.append(entry.cursor_type) - if entry.incremental_mode == IncrementalMode.MICROBATCH: - parts.append("microbatch") - if parts: - return f"{strategy} ({', '.join(parts)})" - return strategy - - -def _snapshot_label(*, entry: ModelPlanEntry, include_prefix: bool) -> str: - parts: list[str] = [] - if entry.snapshot_strategy: - parts.append(entry.snapshot_strategy) - if entry.observed_at_column is not None: - historical_input: str = entry.historical_input or HistoricalInput.SNAPSHOT.value - parts.append(f"historical {historical_input}") - if not parts: - return MaterializationType.SNAPSHOT.value if include_prefix else "" - label: str = ", ".join(parts) - if include_prefix: - return f"snapshot ({label})" - return label diff --git a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py b/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py index 4f13f52740..0d162c1326 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py @@ -4,16 +4,13 @@ from datetime import datetime -from sqlbuild._native import get_config_str, resolve_cursor_input_roles +from sqlbuild._native import aligned_timestamp_bounds, get_config_str, resolve_cursor_input_roles from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompileSqlReference, ) from sqlbuild.compiler.compile.types import CompiledFunctionView, CompiledModelView from sqlbuild.compiler.planner._helpers.resolve.lineage import resolve_lineage_reference -from sqlbuild.compiler.planner.main.execution.aligned_timestamp_bounds import ( - aligned_timestamp_bounds, -) from sqlbuild.compiler.planner.main.execution.microbatch_limit import ( _resolve_microbatch_limit_config, ) diff --git a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_count.py b/src/sqlbuild/compiler/planner/_helpers/output/microbatch_count.py index 8189affa91..247c33d9a8 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_count.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/microbatch_count.py @@ -4,9 +4,7 @@ from datetime import datetime -from sqlbuild.compiler.planner.main.execution.aligned_timestamp_bounds import ( - aligned_timestamp_bounds, -) +from sqlbuild._native import aligned_timestamp_bounds from sqlbuild.compiler.planner.models import CursorBounds, Duration from sqlbuild.compiler.planner.types import CursorType from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index 887ee0c851..eec6e54e65 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -18,7 +18,9 @@ get_config_string_tuple, microbatch_guidance, render_source_relation, + resolve_bounded_cursor_override, resolve_cursor_input_roles, + resolve_effective_microbatch_batch_size, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo @@ -46,9 +48,6 @@ from sqlbuild.compiler.planner._helpers.output.cursor_type_check import ( check_cursor_type_consistency, ) -from sqlbuild.compiler.planner._helpers.output.inclusive_cursor_end import ( - resolve_bounded_cursor_override, -) from sqlbuild.compiler.planner._helpers.output.microbatch_bounds import ( cap_microbatch_bounds, validate_capped_watermark_inputs, @@ -101,9 +100,6 @@ from sqlbuild.compiler.planner.main.changes.relation_replacement import ( replaces_incremental_relation, ) -from sqlbuild.compiler.planner.main.execution.effective_microbatch_batch_size import ( - resolve_effective_microbatch_batch_size, -) from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety from sqlbuild.compiler.planner.main.execution.future_cursor_warning import future_cursor_cap_warning from sqlbuild.compiler.planner.main.execution.maximum_start_warning import maximum_start_cap_warning diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py index c3a7d1450c..c0a19606e0 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py @@ -167,19 +167,6 @@ def empty_cursor_window(*, cursor_type: str | None, cursor_start: str | None) -> return CursorBounds(start=bound, end=bound) -def format_no_input_rows(input_names: tuple[str, ...]) -> str: - """Explain that a cursor window is empty because its inputs have no rows.""" - - return f"no input rows ({', '.join(input_names)})" - - -def format_waiting_on_empty_inputs(input_names: tuple[str, ...]) -> str: - """Explain that a cursor window waits on empty inputs while other inputs have rows.""" - - noun: str = "input" if len(input_names) == 1 else "inputs" - return f"waiting on empty {noun} {', '.join(input_names)}; other inputs have new rows" - - def cursor_window_override_flags(*, cursor_type: str | None) -> str: """Name the CLI flags that set a cursor window explicitly for this cursor type.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py index 96a1043867..eee9f5e4be 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py @@ -9,6 +9,7 @@ assert_no_unresolved_sql_markers, get_config_cursor_bound, get_config_str, + resolve_bounded_cursor_override, resolve_cursor_input_roles, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -20,9 +21,6 @@ ) from sqlbuild.compiler.compile.types import CompiledFunctionView, CompiledModelView from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.planner._helpers.output.inclusive_cursor_end import ( - resolve_bounded_cursor_override, -) from sqlbuild.compiler.planner._helpers.resolve.config import ( get_config_append_cursor_inclusive, ) diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/no_input_rows_display.py b/src/sqlbuild/compiler/planner/main/cursor_window/no_input_rows_display.py deleted file mode 100644 index 2088eba3ce..0000000000 --- a/src/sqlbuild/compiler/planner/main/cursor_window/no_input_rows_display.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public display of cursor inputs that currently have no rows.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - format_no_input_rows as _format_no_input_rows, -) - - -def format_no_input_rows(input_names: tuple[str, ...]) -> str: - """Explain that a cursor window is empty because its inputs have no rows.""" - - return _format_no_input_rows(input_names) diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/waiting_on_empty_inputs_display.py b/src/sqlbuild/compiler/planner/main/cursor_window/waiting_on_empty_inputs_display.py deleted file mode 100644 index 3779375e8b..0000000000 --- a/src/sqlbuild/compiler/planner/main/cursor_window/waiting_on_empty_inputs_display.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public display of a cursor window that waits on empty inputs while others have rows.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - format_waiting_on_empty_inputs as _format_waiting_on_empty_inputs, -) - - -def format_waiting_on_empty_inputs(input_names: tuple[str, ...]) -> str: - """Explain that a cursor window waits on empty inputs while other inputs have rows.""" - - return _format_waiting_on_empty_inputs(input_names) diff --git a/src/sqlbuild/compiler/planner/main/execution/_model_materialization_label.py b/src/sqlbuild/compiler/planner/main/execution/_model_materialization_label.py deleted file mode 100644 index 23ca486f9c..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/_model_materialization_label.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public model materialization label entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.output.materialization_labels import ( - model_materialization_label as _model_materialization_label, -) -from sqlbuild.compiler.planner.models import ModelPlanEntry - - -def model_materialization_label(entry: ModelPlanEntry) -> str: - """Return the full materialization label used in plan summaries.""" - - return _model_materialization_label(entry) diff --git a/src/sqlbuild/compiler/planner/main/execution/aligned_timestamp_bounds.py b/src/sqlbuild/compiler/planner/main/execution/aligned_timestamp_bounds.py deleted file mode 100644 index 5bd37f83a2..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/aligned_timestamp_bounds.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Project temporal cursor bounds onto comparable datetimes.""" - -from datetime import UTC, datetime - -from sqlbuild.cursor_algebra.models import DateValue, TimestampValue - - -def aligned_timestamp_bounds( - *, start: DateValue | TimestampValue, end: DateValue | TimestampValue -) -> tuple[datetime, datetime]: - """Return comparable bounds, presuming a naive side UTC when the other is aware.""" - - start_at: datetime = _as_datetime(value=start) - end_at: datetime = _as_datetime(value=end) - if (start_at.tzinfo is None) != (end_at.tzinfo is None): - start_at = _presume_utc(value=start_at) - end_at = _presume_utc(value=end_at) - return start_at, end_at - - -def _as_datetime(*, value: DateValue | TimestampValue) -> datetime: - if isinstance(value, TimestampValue): - return value.value - return datetime.combine(value.value, datetime.min.time()) - - -def _presume_utc(*, value: datetime) -> datetime: - return value if value.tzinfo is not None else value.replace(tzinfo=UTC) diff --git a/src/sqlbuild/compiler/planner/main/execution/cursor_bound_display.py b/src/sqlbuild/compiler/planner/main/execution/cursor_bound_display.py deleted file mode 100644 index 40dd2e8225..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/cursor_bound_display.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public cursor-bound display entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.output.inclusive_cursor_end import ( - cursor_bound_display as _cursor_bound_display, -) - - -def cursor_bound_display(*, value: str, cursor_type: str | None, cursor_grain: str | None) -> str: - """Render a cursor bound for display, collapsing a whole-day midnight timestamp to a date.""" - - return _cursor_bound_display(value=value, cursor_type=cursor_type, cursor_grain=cursor_grain) diff --git a/src/sqlbuild/compiler/planner/main/execution/effective_microbatch_batch_size.py b/src/sqlbuild/compiler/planner/main/execution/effective_microbatch_batch_size.py deleted file mode 100644 index af4200862c..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/effective_microbatch_batch_size.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Resolve the effective microbatch size after cursor-grain propagation.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.types import CursorGrain -from sqlbuild.cursor_algebra.constants import GRAIN_BATCH_SIZE -from sqlbuild.spec.contracts.constants import EFFECTIVE_BATCH_SIZE_TOKEN - - -def resolve_effective_microbatch_batch_size(*, batch_size: str, effective_grain: str) -> str: - """Resolve the explicit effective batch-size token against the replay grain.""" - - if batch_size != EFFECTIVE_BATCH_SIZE_TOKEN: - return batch_size - return GRAIN_BATCH_SIZE[CursorGrain(effective_grain)] diff --git a/src/sqlbuild/compiler/planner/main/execution/inclusive_cursor_end.py b/src/sqlbuild/compiler/planner/main/execution/inclusive_cursor_end.py deleted file mode 100644 index f8fdf5744f..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/inclusive_cursor_end.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public inclusive cursor end-bound formatting entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.output.inclusive_cursor_end import ( - inclusive_cursor_end as _inclusive_cursor_end, -) - - -def inclusive_cursor_end(*, end: str, cursor_type: str | None, cursor_grain: str | None) -> str: - """Return the last cursor value included by an exclusive end bound.""" - - return _inclusive_cursor_end(end=end, cursor_type=cursor_type, cursor_grain=cursor_grain) diff --git a/src/sqlbuild/compiler/planner/main/execution/materialization_type_display.py b/src/sqlbuild/compiler/planner/main/execution/materialization_type_display.py deleted file mode 100644 index f5636ae316..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/materialization_type_display.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public materialization type display entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.output.materialization_labels import ( - materialization_type_display as _materialization_type_display, -) -from sqlbuild.runtime.contracts.types import ExecutionResourceKind - - -def materialization_type_display(resource_kind: ExecutionResourceKind) -> str: - """Return the left-column resource type for active progress rows.""" - - return _materialization_type_display(resource_kind) diff --git a/src/sqlbuild/compiler/planner/main/execution/model_execution_annotation.py b/src/sqlbuild/compiler/planner/main/execution/model_execution_annotation.py deleted file mode 100644 index 0f41282f43..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/model_execution_annotation.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public model execution annotation entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.output.materialization_labels import ( - model_execution_annotation as _model_execution_annotation, -) -from sqlbuild.compiler.planner.models import ModelPlanEntry - - -def model_execution_annotation(entry: ModelPlanEntry | None) -> str: - """Return the parenthesized model annotation used in execution progress rows.""" - - return _model_execution_annotation(entry) diff --git a/src/sqlbuild/compiler/planner/main/execution/model_resource_type.py b/src/sqlbuild/compiler/planner/main/execution/model_resource_type.py deleted file mode 100644 index 2aa3f786fb..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/model_resource_type.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public model resource type display entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.output.materialization_labels import ( - model_resource_type as _model_resource_type, -) -from sqlbuild.compiler.planner.models import ModelPlanEntry - - -def model_resource_type(entry: ModelPlanEntry | None) -> str: - """Return the left-column resource type used in execution progress rows.""" - - return _model_resource_type(entry) diff --git a/src/sqlbuild/executor/run/_helpers/execution/results.py b/src/sqlbuild/executor/run/_helpers/execution/results.py index d574daa74e..3266c9e301 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/results.py +++ b/src/sqlbuild/executor/run/_helpers/execution/results.py @@ -4,10 +4,8 @@ from dataclasses import replace +from sqlbuild._native import format_waiting_on_empty_inputs from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.planner.main.cursor_window.waiting_on_empty_inputs_display import ( - format_waiting_on_empty_inputs, -) from sqlbuild.compiler.planner.models import ( CursorBounds, FutureCursorSafetyEvidence, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index 6458352071..c5c1454d21 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -11,7 +11,13 @@ from decimal import Decimal, InvalidOperation from typing import Any -from sqlbuild._native import compute_query_hash +from sqlbuild._native import ( + aligned_timestamp_bounds, + compute_query_hash, + cursor_bound_display, + inclusive_cursor_end, + resolve_effective_microbatch_batch_size, +) from sqlbuild._native import cursor_cap_from_end as cap_from_end from sqlbuild._native import cursor_cap_from_start as cap_from_start from sqlbuild._native import cursor_compare as compare @@ -36,15 +42,7 @@ resolve_relation_location_qualified_name, ) from sqlbuild.compiler.planner.main.cursor_window.empty_cursor_window import empty_cursor_window -from sqlbuild.compiler.planner.main.execution.aligned_timestamp_bounds import ( - aligned_timestamp_bounds, -) -from sqlbuild.compiler.planner.main.execution.cursor_bound_display import cursor_bound_display -from sqlbuild.compiler.planner.main.execution.effective_microbatch_batch_size import ( - resolve_effective_microbatch_batch_size, -) from sqlbuild.compiler.planner.main.execution.future_cursor_warning import future_cursor_cap_warning -from sqlbuild.compiler.planner.main.execution.inclusive_cursor_end import inclusive_cursor_end from sqlbuild.compiler.planner.main.execution.maximum_start_warning import maximum_start_cap_warning from sqlbuild.compiler.planner.main.execution.microbatch_limit import ( microbatch_limit_warning, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_cursor_bound_display.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_cursor_bound_display.py index 31037684e8..a21c19ef1b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_cursor_bound_display.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_cursor_bound_display.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.planner._helpers.output.inclusive_cursor_end import cursor_bound_display +from sqlbuild._native import cursor_bound_display from sqlbuild.compiler.planner.types import CursorGrain, CursorType from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( CursorBoundDisplayTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_effective_microbatch_batch_size.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_effective_microbatch_batch_size.py index 7499f59736..f2bb357ec8 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_effective_microbatch_batch_size.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_effective_microbatch_batch_size.py @@ -2,9 +2,7 @@ import pytest -from sqlbuild.compiler.planner.main.execution.effective_microbatch_batch_size import ( - resolve_effective_microbatch_batch_size, -) +from sqlbuild._native import resolve_effective_microbatch_batch_size from sqlbuild.compiler.planner.types import CursorGrain from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( EffectiveMicrobatchBatchSizeTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_inclusive_cursor_end.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_inclusive_cursor_end.py index 5d9f6087be..279f4c809e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_inclusive_cursor_end.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_inclusive_cursor_end.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.planner.main.execution.inclusive_cursor_end import inclusive_cursor_end +from sqlbuild._native import inclusive_cursor_end from sqlbuild.compiler.planner.types import CursorGrain, CursorType from tests.unit.src.sqlbuild.compiler.planner.main._test_types import InclusiveCursorEndTestCase From fee4fc238cd5f9a0f7d384af7a4247d75443ecfc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 01:07:45 +0100 Subject: [PATCH 10/24] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 154d97d82b..dbc3ea92c2 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 67450 +remaining_lines = 67089 [[stays]] path = "executor" From c2fa77a4dc7a56b8686455a325c2975d3558890b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:51:28 +0100 Subject: [PATCH 11/24] refactor: parse dbt event streams natively --- .../src/dbt_events/_helpers/mod.rs | 2 + .../src/dbt_events/_helpers/parse.rs | 114 +++++ .../src/dbt_events/_helpers/rows.rs | 31 ++ .../sqlbuild-core/src/dbt_events/constants.rs | 46 ++ .../src/dbt_events/main/consume_dbt_event.rs | 72 +++ .../main/format_dbt_active_nodes.rs | 39 ++ .../sqlbuild-core/src/dbt_events/main/mod.rs | 3 + .../dbt_events/main/render_dbt_node_row.rs | 62 +++ crates/sqlbuild-core/src/dbt_events/mod.rs | 9 + crates/sqlbuild-core/src/dbt_events/models.rs | 89 ++++ crates/sqlbuild-core/src/dbt_events/tests.rs | 6 + .../src/dbt_events/tests/helpers.rs | 105 +++++ .../src/dbt_events/tests/test_dbt_events.rs | 131 ++++++ .../src/dbt_events/tests/test_types.rs | 13 + crates/sqlbuild-core/src/lib.rs | 1 + .../src/bindings/_helpers/cli/registration.rs | 7 +- .../src/bindings/_helpers/dbt_events/event.rs | 69 +++ .../src/bindings/_helpers/dbt_events/mod.rs | 4 + .../_helpers/dbt_events/registration.rs | 9 + .../bindings/_helpers/dbt_events/results.rs | 110 +++++ .../bindings/_helpers/dbt_events/stream.rs | 158 +++++++ .../src/bindings/_helpers/mod.rs | 1 + python-boundary.toml | 10 + src/sqlbuild/_native.pyi | 33 ++ .../dbt/_helpers/pipeline/execute.py | 17 +- .../dbt/_helpers/runtime/event_stream.py | 416 ++---------------- .../dbt/main/runtime/_render_node_result.py | 30 -- .../src/sqlbuild/integrations/dbt/helpers.py | 19 +- .../integrations/dbt/test_event_stream.py | 86 ++-- 29 files changed, 1243 insertions(+), 449 deletions(-) create mode 100644 crates/sqlbuild-core/src/dbt_events/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/dbt_events/_helpers/parse.rs create mode 100644 crates/sqlbuild-core/src/dbt_events/_helpers/rows.rs create mode 100644 crates/sqlbuild-core/src/dbt_events/constants.rs create mode 100644 crates/sqlbuild-core/src/dbt_events/main/consume_dbt_event.rs create mode 100644 crates/sqlbuild-core/src/dbt_events/main/format_dbt_active_nodes.rs create mode 100644 crates/sqlbuild-core/src/dbt_events/main/mod.rs create mode 100644 crates/sqlbuild-core/src/dbt_events/main/render_dbt_node_row.rs create mode 100644 crates/sqlbuild-core/src/dbt_events/mod.rs create mode 100644 crates/sqlbuild-core/src/dbt_events/models.rs create mode 100644 crates/sqlbuild-core/src/dbt_events/tests.rs create mode 100644 crates/sqlbuild-core/src/dbt_events/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/dbt_events/tests/test_dbt_events.rs create mode 100644 crates/sqlbuild-core/src/dbt_events/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_events/event.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_events/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_events/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_events/results.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs delete mode 100644 src/sqlbuild/integrations/dbt/main/runtime/_render_node_result.py diff --git a/crates/sqlbuild-core/src/dbt_events/_helpers/mod.rs b/crates/sqlbuild-core/src/dbt_events/_helpers/mod.rs new file mode 100644 index 0000000000..97d768e678 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_events/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod parse; +pub(crate) mod rows; diff --git a/crates/sqlbuild-core/src/dbt_events/_helpers/parse.rs b/crates/sqlbuild-core/src/dbt_events/_helpers/parse.rs new file mode 100644 index 0000000000..0e65ac27b6 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_events/_helpers/parse.rs @@ -0,0 +1,114 @@ +//! Reading node messages, starts and results from dbt events. + +use crate::dbt_events::constants::{ + DEFAULT_MESSAGE_LEVEL, DEFAULT_RESOURCE_TYPE, NODE_FINISHED_EVENT, NODE_MESSAGE_EVENT_NAMES, + NODE_MESSAGE_LEVELS, NODE_STARTED_EVENT_NAMES, OUTCOME_STATUSES, RESULT_EVENT_NAMES, + START_STATUS, UNKNOWN_STATUS, +}; +use crate::dbt_events::models::{DbtEvent, DbtNodeMessage, DbtNodeResult}; +use crate::text::main::python_strip::python_strip; + +/// Non-empty text, as dbt fields are read. +pub(crate) fn present(value: Option<&String>) -> Option { + value.filter(|text| !text.is_empty()).cloned() +} + +fn named(event: &DbtEvent, names: &[&str]) -> bool { + present(event.name.as_ref()).is_some_and(|name| names.contains(&name.as_str())) +} + +/// Python's `total or num_models`. +fn event_total(event: &DbtEvent) -> Option { + match event.total { + Some(total) if total != 0 => Some(total), + _ => event.num_models, + } +} + +/// The node's unique id, when the event names one. +pub(crate) fn unique_id(event: &DbtEvent) -> Option { + present(event.node.unique_id.as_ref()) +} + +/// A node-scoped warning or error message. +pub(crate) fn node_message(event: &DbtEvent) -> Option { + let level: Option = present(event.level.as_ref()); + let level_matches: bool = level + .as_ref() + .is_some_and(|level| NODE_MESSAGE_LEVELS.contains(&level.as_str())); + if !named(event, &NODE_MESSAGE_EVENT_NAMES) && !level_matches { + return None; + } + unique_id(event)?; + let message: String = + present(event.data_msg.as_ref()).or_else(|| present(event.info_msg.as_ref()))?; + Some(DbtNodeMessage { + level: level.unwrap_or_else(|| DEFAULT_MESSAGE_LEVEL.to_owned()), + message: python_strip(&message).to_owned(), + }) +} + +fn base_result(event: &DbtEvent, unique_id: String, status: String) -> DbtNodeResult { + DbtNodeResult { + resource_type: present(event.node.resource_type.as_ref()) + .unwrap_or_else(|| DEFAULT_RESOURCE_TYPE.to_owned()), + node_name: present(event.node.node_name.as_ref()).unwrap_or_else(|| unique_id.clone()), + unique_id, + status, + index: event.index, + total: event_total(event), + materialized: present(event.node.materialized.as_ref()), + ..DbtNodeResult::default() + } +} + +/// A START row for a node-started event. +pub(crate) fn node_start(event: &DbtEvent) -> Option { + if !named(event, &NODE_STARTED_EVENT_NAMES) { + return None; + } + Some(base_result( + event, + unique_id(event)?, + START_STATUS.to_owned(), + )) +} + +/// The progress message for a node-started event. +pub(crate) fn start_message(result: &DbtNodeResult) -> String { + format!( + "Running dbt {} {}...", + result.resource_type, result.node_name + ) +} + +fn trusted_status(value: Option<&String>) -> Option { + present(value).filter(|status| OUTCOME_STATUSES.contains(&status.to_lowercase().as_str())) +} + +/// A final node result, without its buffered messages. +pub(crate) fn node_result(event: &DbtEvent) -> Option { + if !named(event, &RESULT_EVENT_NAMES) { + return None; + } + let unique_id: String = unique_id(event)?; + let node_status: Option = present(event.node.node_status.as_ref()); + let status: Option = + if present(event.name.as_ref()).as_deref() == Some(NODE_FINISHED_EVENT) { + present(event.run_result_status.as_ref()).or(node_status) + } else { + trusted_status(event.data_status.as_ref()).or(node_status) + }; + Some(DbtNodeResult { + execution_time: event.execution_time, + relation_name: present(event.node.relation_name.as_ref()), + database: present(event.node.database.as_ref()), + schema: present(event.node.schema.as_ref()), + node_checksum: present(event.node.node_checksum.as_ref()), + ..base_result( + event, + unique_id, + status.unwrap_or_else(|| UNKNOWN_STATUS.to_owned()), + ) + }) +} diff --git a/crates/sqlbuild-core/src/dbt_events/_helpers/rows.rs b/crates/sqlbuild-core/src/dbt_events/_helpers/rows.rs new file mode 100644 index 0000000000..24b86d7d67 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_events/_helpers/rows.rs @@ -0,0 +1,31 @@ +//! Node row columns: counter, status label and truncated text. + +use crate::dbt_events::constants::DISPLAY_STATUSES; +use crate::dbt_events::models::{DbtNodeResult, DbtRowPosition}; + +/// The stream counter, else dbt's index and total, else the stream index alone. +pub(crate) fn counter(result: &DbtNodeResult, position: DbtRowPosition) -> String { + if let (Some(index), Some(total)) = (position.display_index, position.display_total) { + return format!("{index}/{total}"); + } + if let (Some(index), Some(total)) = (result.index, result.total) { + return format!("{index}/{total}"); + } + position + .display_index + .map_or_else(|| "-".to_owned(), |index| index.to_string()) +} + +/// The label for a dbt status, else the status upper-cased. +pub(crate) fn display_status(status: &str) -> String { + let normalized: String = status.to_lowercase(); + DISPLAY_STATUSES + .iter() + .find(|(statuses, _)| statuses.contains(&normalized.as_str())) + .map_or_else(|| status.to_uppercase(), |(_, label)| (*label).to_owned()) +} + +/// The first `width` characters. +pub(crate) fn truncated(text: &str, width: usize) -> String { + text.chars().take(width).collect() +} diff --git a/crates/sqlbuild-core/src/dbt_events/constants.rs b/crates/sqlbuild-core/src/dbt_events/constants.rs new file mode 100644 index 0000000000..bc05134c84 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_events/constants.rs @@ -0,0 +1,46 @@ +//! dbt event names, levels and statuses the stream reads. + +pub(crate) const NODE_FINISHED_EVENT: &str = "NodeFinished"; +pub(crate) const NODE_MESSAGE_EVENT_NAMES: [&str; 4] = [ + "RunResultError", + "RunResultFailure", + "RunResultWarning", + "GenericExceptionOnRun", +]; +pub(crate) const NODE_MESSAGE_LEVELS: [&str; 2] = ["warn", "error"]; +pub(crate) const NODE_STARTED_EVENT_NAMES: [&str; 2] = ["LogStartLine", "NodeStarted"]; +pub(crate) const RESULT_EVENT_NAMES: [&str; 7] = [ + "LogModelResult", + "LogSeedResult", + "LogSnapshotResult", + "LogTestResult", + "LogBatchResult", + "LogFunctionResult", + "NodeFinished", +]; +pub(crate) const OUTCOME_STATUSES: [&str; 11] = [ + "error", "fail", "failed", "ok", "pass", "passed", "skip", "skipped", "success", "warn", + "warning", +]; +pub(crate) const START_STATUS: &str = "start"; +pub(crate) const UNKNOWN_STATUS: &str = "unknown"; +pub(crate) const DEFAULT_RESOURCE_TYPE: &str = "node"; +pub(crate) const DEFAULT_MESSAGE_LEVEL: &str = "info"; +pub(crate) const WARN_LEVEL: &str = "warn"; +pub(crate) const ERROR_LEVEL: &str = "error"; +/// Display labels by the dbt statuses that map to them. +pub(crate) const DISPLAY_STATUSES: [(&[&str], &str); 6] = [ + (&["ok", "success"], "OK"), + (&["pass", "passed"], "PASS"), + (&["warn", "warning"], "WARN"), + (&["skip", "skipped"], "SKIP"), + (&["error", "fail", "failed"], "FAIL"), + (&["start"], "START"), +]; +pub(crate) const RESOURCE_TYPE_WIDTH: usize = 9; +pub(crate) const NAME_WIDTH: usize = 30; +pub(crate) const COUNTER_WIDTH: usize = 5; +pub(crate) const STATUS_WIDTH: usize = 6; +pub(crate) const DURATION_WIDTH: usize = 7; +pub(crate) const MESSAGE_STATUS_WIDTH: usize = 9; +pub(crate) const MAX_LISTED_ACTIVE_NODES: usize = 3; diff --git a/crates/sqlbuild-core/src/dbt_events/main/consume_dbt_event.rs b/crates/sqlbuild-core/src/dbt_events/main/consume_dbt_event.rs new file mode 100644 index 0000000000..f348c24fe7 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_events/main/consume_dbt_event.rs @@ -0,0 +1,72 @@ +//! Advance the dbt stream by one event. + +use crate::dbt_events::_helpers::parse::{ + node_message, node_result, node_start, start_message, unique_id, +}; +use crate::dbt_events::models::{DbtEvent, DbtStreamRow, DbtStreamState, DbtStreamUpdate}; + +/// Buffer node messages, number started nodes, and report each node's first final result once. +pub fn consume_dbt_event( + mut state: DbtStreamState, + event: &DbtEvent, +) -> (DbtStreamState, Option) { + if let Some(message) = node_message(event) { + if let Some(unique_id) = unique_id(event) { + state + .pending_messages + .entry(unique_id) + .or_default() + .push(message); + } + return (state, None); + } + if let Some(start) = node_start(event) { + let status_message: Option = Some(start_message(&start)); + if state.started_indexes.contains_key(&start.unique_id) { + let update: DbtStreamUpdate = DbtStreamUpdate { + status_message, + row: None, + }; + return (state, Some(update)); + } + state.display_index += 1; + state + .started_indexes + .insert(start.unique_id.clone(), state.display_index); + let update: DbtStreamUpdate = DbtStreamUpdate { + status_message, + row: Some(DbtStreamRow { + result: start, + display_index: state.display_index, + finished: false, + }), + }; + return (state, Some(update)); + } + let Some(mut result) = node_result(event) else { + return (state, None); + }; + result.messages = state + .pending_messages + .remove(&result.unique_id) + .unwrap_or_default(); + if !state.recorded_unique_ids.insert(result.unique_id.clone()) { + return (state, None); + } + let display_index: i64 = match state.started_indexes.get(&result.unique_id) { + Some(index) => *index, + None => { + state.display_index += 1; + state.display_index + } + }; + let update: DbtStreamUpdate = DbtStreamUpdate { + status_message: None, + row: Some(DbtStreamRow { + result, + display_index, + finished: true, + }), + }; + (state, Some(update)) +} diff --git a/crates/sqlbuild-core/src/dbt_events/main/format_dbt_active_nodes.rs b/crates/sqlbuild-core/src/dbt_events/main/format_dbt_active_nodes.rs new file mode 100644 index 0000000000..cdea5fd9e4 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_events/main/format_dbt_active_nodes.rs @@ -0,0 +1,39 @@ +//! The progress line listing the dbt nodes still running. + +use crate::dbt_events::constants::MAX_LISTED_ACTIVE_NODES; + +fn elapsed(now: f64, started_at: f64) -> String { + let seconds: i64 = ((now - started_at) as i64).max(0); + if seconds == 0 { + return "<1s".to_owned(); + } + format!("{seconds}s") +} + +/// The running count and the three longest-running nodes with their elapsed time. +#[must_use] +pub fn format_dbt_active_nodes(active_nodes: &[(String, f64)], now: f64) -> String { + let mut ordered: Vec<&(String, f64)> = active_nodes.iter().collect(); + ordered.sort_by(|left, right| left.1.total_cmp(&right.1)); + let displayed: Vec = ordered + .iter() + .take(MAX_LISTED_ACTIVE_NODES) + .map(|(name, started_at)| format!("{name} {}", elapsed(now, *started_at))) + .collect(); + let extra: usize = ordered.len() - displayed.len(); + let suffix: String = if extra > 0 { + format!(", +{extra} more") + } else { + String::new() + }; + let label: &str = if active_nodes.len() == 1 { + "node" + } else { + "nodes" + }; + format!( + "running {} dbt {label}: {}{suffix}", + active_nodes.len(), + displayed.join(", ") + ) +} diff --git a/crates/sqlbuild-core/src/dbt_events/main/mod.rs b/crates/sqlbuild-core/src/dbt_events/main/mod.rs new file mode 100644 index 0000000000..fe40c9749a --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_events/main/mod.rs @@ -0,0 +1,3 @@ +pub mod consume_dbt_event; +pub mod format_dbt_active_nodes; +pub mod render_dbt_node_row; diff --git a/crates/sqlbuild-core/src/dbt_events/main/render_dbt_node_row.rs b/crates/sqlbuild-core/src/dbt_events/main/render_dbt_node_row.rs new file mode 100644 index 0000000000..4d76fb8d48 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_events/main/render_dbt_node_row.rs @@ -0,0 +1,62 @@ +//! One dbt node row and its messages. + +use crate::cli_text::main::format_aligned_name_value::format_aligned_name_value; +use crate::cli_text::models::{AlignedRow, CliStyle}; +use crate::dbt_events::_helpers::rows::{counter, display_status, truncated}; +use crate::dbt_events::constants::{ + COUNTER_WIDTH, DURATION_WIDTH, ERROR_LEVEL, MESSAGE_STATUS_WIDTH, NAME_WIDTH, + RESOURCE_TYPE_WIDTH, STATUS_WIDTH, WARN_LEVEL, +}; +use crate::dbt_events::models::{DbtNodeResult, DbtRowPosition}; +use crate::text::main::python_isspace::python_isspace; + +/// The row text with a trailing newline, then one line per node message. +#[must_use] +pub fn render_dbt_node_row( + result: &DbtNodeResult, + position: DbtRowPosition, + detail: &str, + style: CliStyle, +) -> String { + let status: String = display_status(&result.status); + let duration: String = result + .execution_time + .map_or_else(String::new, |seconds| format!("{seconds:.2}s")); + let padding: String = " ".repeat(STATUS_WIDTH.saturating_sub(status.chars().count())); + let rendered_detail: String = if detail.is_empty() { + String::new() + } else { + style.muted(detail) + }; + let value: String = format!( + "{}{padding} {duration:, + pub resource_type: Option, + pub node_name: Option, + pub materialized: Option, + pub node_status: Option, + pub node_checksum: Option, + pub relation_name: Option, + pub database: Option, + pub schema: Option, +} + +/// The fields of one dbt JSON log line the stream reads. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DbtEvent { + pub name: Option, + pub level: Option, + pub info_msg: Option, + pub data_msg: Option, + pub data_status: Option, + pub run_result_status: Option, + pub index: Option, + pub total: Option, + pub num_models: Option, + pub execution_time: Option, + pub node: DbtEventNode, +} + +/// One warning or error dbt logged for a node. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DbtNodeMessage { + pub level: String, + pub message: String, +} + +/// One dbt node start or final result. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DbtNodeResult { + pub unique_id: String, + pub resource_type: String, + pub node_name: String, + pub status: String, + pub index: Option, + pub total: Option, + pub execution_time: Option, + pub materialized: Option, + pub relation_name: Option, + pub database: Option, + pub schema: Option, + pub node_checksum: Option, + pub messages: Vec, +} + +/// What the stream has seen so far: buffered messages, started and finished nodes. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DbtStreamState { + pub pending_messages: HashMap>, + pub recorded_unique_ids: HashSet, + pub started_indexes: HashMap, + pub display_index: i64, +} + +/// A node row to write: a start row, or a final result. +#[derive(Clone, Debug, PartialEq)] +pub struct DbtStreamRow { + pub result: DbtNodeResult, + pub display_index: i64, + pub finished: bool, +} + +/// How one event changes the output: a progress message, a row, or both. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DbtStreamUpdate { + pub status_message: Option, + pub row: Option, +} + +/// Where a node row sits: the stream counter, else dbt's own index and total. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DbtRowPosition { + pub display_index: Option, + pub display_total: Option, +} diff --git a/crates/sqlbuild-core/src/dbt_events/tests.rs b/crates/sqlbuild-core/src/dbt_events/tests.rs new file mode 100644 index 0000000000..024511b046 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_events/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_dbt_events.rs"] +mod dbt_events; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/dbt_events/tests/helpers.rs b/crates/sqlbuild-core/src/dbt_events/tests/helpers.rs new file mode 100644 index 0000000000..b435bc9291 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_events/tests/helpers.rs @@ -0,0 +1,105 @@ +use crate::dbt_events::main::consume_dbt_event::consume_dbt_event; +use crate::dbt_events::models::{ + DbtEvent, DbtEventNode, DbtNodeMessage, DbtNodeResult, DbtStreamRow, DbtStreamState, + DbtStreamUpdate, +}; + +fn text(value: &str) -> Option { + Some(value.to_owned()) +} + +/// A dbt event named `name` for the node `unique_id`. +pub(super) fn event(name: &str, unique_id: &str) -> DbtEvent { + DbtEvent { + name: text(name), + level: text("info"), + node: DbtEventNode { + unique_id: text(unique_id), + resource_type: text("model"), + node_name: text(unique_id.rsplit('.').next().unwrap_or_default()), + ..DbtEventNode::default() + }, + ..DbtEvent::default() + } +} + +/// A final model result event with a status, timing and counter. +pub(super) fn result_event(name: &str, unique_id: &str, status: &str) -> DbtEvent { + DbtEvent { + data_status: text(status), + index: Some(1), + total: Some(0), + num_models: Some(2), + execution_time: Some(0.125), + ..event(name, unique_id) + } +} + +/// An error message event for the node `unique_id`. +pub(super) fn message_event(unique_id: &str, message: &str) -> DbtEvent { + DbtEvent { + level: text("error"), + data_msg: text(""), + info_msg: text(message), + ..event("RunResultError", unique_id) + } +} + +/// The updates a fresh stream reports for `events`. +pub(super) fn updates(events: &[DbtEvent]) -> Vec> { + let mut state: DbtStreamState = DbtStreamState::default(); + let mut updates: Vec> = Vec::new(); + for event in events { + let (next, update) = consume_dbt_event(state, event); + state = next; + updates.push(update); + } + updates +} + +/// The update for a node-started event at `display_index`. +pub(super) fn started(unique_id: &str, display_index: i64) -> Option { + Some(DbtStreamUpdate { + status_message: Some(format!( + "Running dbt model {}...", + unique_id.rsplit('.').next().unwrap_or_default() + )), + row: Some(DbtStreamRow { + result: DbtNodeResult { + unique_id: unique_id.to_owned(), + resource_type: "model".to_owned(), + node_name: unique_id.rsplit('.').next().unwrap_or_default().to_owned(), + status: "start".to_owned(), + ..DbtNodeResult::default() + }, + display_index, + finished: false, + }), + }) +} + +/// The update for a final model result. +pub(super) fn finished( + unique_id: &str, + (status, display_index): (&str, i64), + messages: Vec, +) -> Option { + Some(DbtStreamUpdate { + status_message: None, + row: Some(DbtStreamRow { + result: DbtNodeResult { + unique_id: unique_id.to_owned(), + resource_type: "model".to_owned(), + node_name: unique_id.rsplit('.').next().unwrap_or_default().to_owned(), + status: status.to_owned(), + index: Some(1), + total: Some(2), + execution_time: Some(0.125), + messages, + ..DbtNodeResult::default() + }, + display_index, + finished: true, + }), + }) +} diff --git a/crates/sqlbuild-core/src/dbt_events/tests/test_dbt_events.rs b/crates/sqlbuild-core/src/dbt_events/tests/test_dbt_events.rs new file mode 100644 index 0000000000..6d052de54b --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_events/tests/test_dbt_events.rs @@ -0,0 +1,131 @@ +use crate::cli_text::models::CliStyle; +use crate::dbt_events::main::format_dbt_active_nodes::format_dbt_active_nodes; +use crate::dbt_events::main::render_dbt_node_row::render_dbt_node_row; +use crate::dbt_events::models::{ + DbtEvent, DbtNodeMessage, DbtNodeResult, DbtRowPosition, DbtStreamUpdate, +}; +use crate::dbt_events::tests::helpers::{ + event, finished, message_event, result_event, started, updates, +}; +use crate::dbt_events::tests::test_types::{StreamTestCase, TextTestCase}; + +const PLAIN: CliStyle = CliStyle { use_color: false }; + +#[test] +fn given_dbt_events_when_consuming_then_reports_starts_and_first_results() { + let test_cases = [ + StreamTestCase { + description: "a start row, its result at the same counter, then a duplicate", + events: vec![ + event("NodeStarted", "model.a.orders"), + event("NodeStarted", "model.a.orders"), + result_event("LogModelResult", "model.a.orders", "SELECT 1"), + result_event("NodeFinished", "model.a.orders", "success"), + ], + expected_updates: vec![ + started("model.a.orders", 1), + Some(DbtStreamUpdate { + row: None, + ..started("model.a.orders", 1).unwrap_or_default() + }), + finished("model.a.orders", ("unknown", 1), Vec::new()), + None, + ], + }, + StreamTestCase { + description: "buffered messages attach to the result and other lines are ignored", + events: vec![ + message_event("model.a.customers", " relation missing \n"), + DbtEvent::default(), + event("FoundStats", "model.a.customers"), + result_event("LogModelResult", "model.a.customers", "ERROR"), + ], + expected_updates: vec![ + None, + None, + None, + finished( + "model.a.customers", + ("ERROR", 1), + vec![DbtNodeMessage { + level: "error".to_owned(), + message: "relation missing".to_owned(), + }], + ), + ], + }, + ]; + + for test_case in test_cases { + assert_eq!( + updates(&test_case.events), + test_case.expected_updates, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_node_rows_when_rendering_then_writes_aligned_rows_and_progress() { + let result: DbtNodeResult = DbtNodeResult { + unique_id: "model.a.orders".to_owned(), + resource_type: "incremental_model".to_owned(), + node_name: "orders".to_owned(), + status: "success".to_owned(), + index: Some(3), + total: Some(9), + execution_time: Some(1.005), + messages: vec![DbtNodeMessage { + level: "warn".to_owned(), + message: "slow".to_owned(), + }], + ..DbtNodeResult::default() + }; + let test_cases = [ + TextTestCase { + description: "dbt counter, truncated type, duration and message", + rendered: render_dbt_node_row(&result, DbtRowPosition::default(), "", PLAIN), + expected_text: " 3/9 increment orders OK 1.00s\n warn slow\n", + }, + TextTestCase { + description: "stream counter and detail without timing", + rendered: render_dbt_node_row( + &DbtNodeResult { + execution_time: None, + messages: Vec::new(), + status: "start".to_owned(), + ..result.clone() + }, + DbtRowPosition { + display_index: Some(2), + display_total: Some(4), + }, + "defer", + PLAIN, + ), + expected_text: " 2/4 increment orders START defer\n", + }, + TextTestCase { + description: "longest-running nodes first, with the rest counted", + rendered: format_dbt_active_nodes( + &[ + ("c".to_owned(), 9.5), + ("a".to_owned(), 1.0), + ("b".to_owned(), 5.0), + ("d".to_owned(), 9.9), + ], + 10.0, + ), + expected_text: "running 4 dbt nodes: a 9s, b 5s, c <1s, +1 more", + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_case.rendered, test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/dbt_events/tests/test_types.rs b/crates/sqlbuild-core/src/dbt_events/tests/test_types.rs new file mode 100644 index 0000000000..099e1db152 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_events/tests/test_types.rs @@ -0,0 +1,13 @@ +use crate::dbt_events::models::{DbtEvent, DbtStreamUpdate}; + +pub(super) struct StreamTestCase { + pub(super) description: &'static str, + pub(super) events: Vec, + pub(super) expected_updates: Vec>, +} + +pub(super) struct TextTestCase { + pub(super) description: &'static str, + pub(super) rendered: String, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 1bc27fe670..d3d66a0ae7 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -12,6 +12,7 @@ pub mod contract_command; pub mod cost_command; pub mod cost_report; pub mod dbt_command; +pub mod dbt_events; pub mod debug_report; pub mod format_command; pub mod freshness_text; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 198f1e8e85..58be3c4671 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -5,9 +5,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::cli::{argv, entry, progress, progress_reporters, progress_text}; use crate::bindings::_helpers::{ audit_command, build_command, check, clone, command_runtime, contract_command, cost_command, - cost_report, dbt_command, dbt_interop, debug_command, diff_report, format_command, freshness, - init_command, janitor, load, presentation_text, query_command, result_output, scenario_command, - seed_command, sql_test_command, workspace_commands, + cost_report, dbt_command, dbt_events, dbt_interop, debug_command, diff_report, format_command, + freshness, init_command, janitor, load, presentation_text, query_command, result_output, + scenario_command, seed_command, sql_test_command, workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -23,6 +23,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { result_output::registration::register(module)?; cost_report::registration::register(module)?; debug_command::registration::register(module)?; + dbt_events::registration::register(module)?; dbt_interop::registration::register(module)?; diff_report::registration::register(module)?; command_runtime::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/event.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/event.rs new file mode 100644 index 0000000000..e431342969 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/event.rs @@ -0,0 +1,69 @@ +//! The fields of one dbt JSON log line, read with Python `dict.get` semantics. + +use pyo3::PyResult; +use pyo3::exceptions::PyRuntimeError; +use serde_json::{Map, Value}; +use sqlbuild_core::dbt_events::models::{DbtEvent, DbtEventNode}; +use sqlbuild_core::text::main::python_strip::python_strip; + +static EMPTY: std::sync::LazyLock> = std::sync::LazyLock::new(Map::new); + +/// The object under `key`, or an empty one. +fn object<'a>(parent: &'a Map, key: &str) -> &'a Map { + parent.get(key).and_then(Value::as_object).unwrap_or(&EMPTY) +} + +fn text(parent: &Map, key: &str) -> Option { + parent.get(key).and_then(Value::as_str).map(str::to_owned) +} + +fn integer(parent: &Map, key: &str) -> Option { + parent.get(key).and_then(Value::as_i64) +} + +fn node(data: &Map) -> DbtEventNode { + let info: &Map = object(data, "node_info"); + let relation: &Map = object(info, "node_relation"); + DbtEventNode { + unique_id: text(info, "unique_id"), + resource_type: text(info, "resource_type"), + node_name: text(info, "node_name"), + materialized: text(info, "materialized"), + node_status: text(info, "node_status"), + node_checksum: text(info, "node_checksum"), + relation_name: text(relation, "relation_name"), + database: text(relation, "database"), + schema: text(relation, "schema"), + } +} + +/// The event on one output line, or `None` for non-JSON output and non-object JSON. +pub(crate) fn dbt_event(line: &str) -> PyResult> { + let stripped: &str = python_strip(line); + if !stripped.starts_with('{') { + return Ok(None); + } + let payload: Value = match serde_json::from_str(stripped) { + Ok(payload) => payload, + Err(error) if error.is_syntax() || error.is_eof() || error.is_data() => return Ok(None), + Err(error) => return Err(PyRuntimeError::new_err(error.to_string())), + }; + let Some(event) = payload.as_object() else { + return Ok(None); + }; + let info: &Map = object(event, "info"); + let data: &Map = object(event, "data"); + Ok(Some(DbtEvent { + name: text(info, "name"), + level: text(info, "level"), + info_msg: text(info, "msg"), + data_msg: text(data, "msg"), + data_status: text(data, "status"), + run_result_status: text(object(data, "run_result"), "status"), + index: integer(data, "index"), + total: integer(data, "total"), + num_models: integer(data, "num_models"), + execution_time: data.get("execution_time").and_then(Value::as_f64), + node: node(data), + })) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/mod.rs new file mode 100644 index 0000000000..ffde820035 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod event; +pub(crate) mod registration; +pub(crate) mod results; +pub(crate) mod stream; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/registration.rs new file mode 100644 index 0000000000..b6e7d9653f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/registration.rs @@ -0,0 +1,9 @@ +//! Register the native dbt event stream. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::dbt_events::stream; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + stream::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/results.rs new file mode 100644 index 0000000000..80858efe42 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/results.rs @@ -0,0 +1,110 @@ +//! dbt node results as the Python `DbtNodeExecutionResult` model. + +use pyo3::BoundObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyString, PyTuple}; +use sqlbuild_core::dbt_events::models::{DbtNodeMessage, DbtNodeResult}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; + +fn optional<'py, T: pyo3::IntoPyObject<'py>>( + py: Python<'py>, + value: Option, +) -> PyResult> { + match value { + Some(value) => Ok(value + .into_pyobject(py) + .map_err(Into::into)? + .into_any() + .into_bound()), + None => Ok(py.None().into_bound(py)), + } +} + +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +/// A Python `DbtNodeExecutionResult`. +pub(crate) fn python_result<'py>( + py: Python<'py>, + result: &DbtNodeResult, +) -> PyResult> { + let message_class: Bound<'py, PyAny> = imported(py, DBT_MODELS, "DbtNodeMessage")?; + let messages: Vec> = result + .messages + .iter() + .map(|message| { + call_with( + &message_class, + &[ + ("level", text(py, &message.level)), + ("message", text(py, &message.message)), + ], + ) + }) + .collect::>()?; + call_with( + &imported(py, DBT_MODELS, "DbtNodeExecutionResult")?, + &[ + ("unique_id", text(py, &result.unique_id)), + ("resource_type", text(py, &result.resource_type)), + ("node_name", text(py, &result.node_name)), + ("status", text(py, &result.status)), + ("index", optional(py, result.index)?), + ("total", optional(py, result.total)?), + ("execution_time", optional(py, result.execution_time)?), + ( + "materialized", + optional(py, result.materialized.as_deref())?, + ), + ( + "relation_name", + optional(py, result.relation_name.as_deref())?, + ), + ("database", optional(py, result.database.as_deref())?), + ("schema", optional(py, result.schema.as_deref())?), + ( + "node_checksum", + optional(py, result.node_checksum.as_deref())?, + ), + ("messages", PyTuple::new(py, messages)?.into_any()), + ], + ) +} + +fn field pyo3::FromPyObject<'a>>(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + value.getattr(name)?.extract() +} + +/// A Python `DbtNodeExecutionResult` as plain data. +pub(crate) fn native_result(result: &Bound<'_, PyAny>) -> PyResult { + let messages: Vec = result + .getattr("messages")? + .try_iter()? + .map(|message| { + let message: Bound<'_, PyAny> = message?; + Ok(DbtNodeMessage { + level: field(&message, "level")?, + message: field(&message, "message")?, + }) + }) + .collect::>()?; + Ok(DbtNodeResult { + unique_id: field(result, "unique_id")?, + resource_type: field(result, "resource_type")?, + node_name: field(result, "node_name")?, + status: field(result, "status")?, + index: field(result, "index")?, + total: field(result, "total")?, + execution_time: field(result, "execution_time")?, + materialized: field(result, "materialized")?, + relation_name: field(result, "relation_name")?, + database: field(result, "database")?, + schema: field(result, "schema")?, + node_checksum: field(result, "node_checksum")?, + messages, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs new file mode 100644 index 0000000000..7ccd60218c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs @@ -0,0 +1,158 @@ +//! The dbt event stream: one output line in, the progress and row it adds out. + +use std::sync::Mutex; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyDict; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::dbt_events::main::consume_dbt_event::consume_dbt_event; +use sqlbuild_core::dbt_events::main::format_dbt_active_nodes::format_dbt_active_nodes; +use sqlbuild_core::dbt_events::main::render_dbt_node_row::render_dbt_node_row; +use sqlbuild_core::dbt_events::models::{DbtRowPosition, DbtStreamState, DbtStreamUpdate}; + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::dbt_events::event::dbt_event; +use crate::bindings::_helpers::dbt_events::results::{native_result, python_result}; + +const STREAM_OPTIONS: (&[&str], &[&str]) = + (&["display_total", "detail_by_unique_id", "use_color"], &[]); +const RENDER_OPTIONS: (&[&str], &[&str]) = ( + &["result", "use_color"], + &["display_index", "display_total", "detail"], +); + +/// What one dbt output line adds: a progress message, a node row, or both. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +pub(crate) struct DbtEventStreamUpdate { + /// The node-started progress message. + status_message: Option, + /// The started or finished node. + result: Option>, + /// The rendered row and its messages. + row: Option, + /// Whether `result` is a final result rather than a start. + finished: bool, +} + +/// Read dbt's JSON log lines into node progress, rows and results. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct DbtEventStream { + state: Mutex, + display_total: Option, + detail_by_unique_id: Py, + style: CliStyle, +} + +impl DbtEventStream { + fn detail(&self, py: Python<'_>, unique_id: &str) -> PyResult { + let details: &Bound<'_, PyAny> = self.detail_by_unique_id.bind(py); + if details.is_none() { + return Ok(String::new()); + } + details.call_method1("get", (unique_id, ""))?.extract() + } + + fn update(&self, py: Python<'_>, update: DbtStreamUpdate) -> PyResult { + let Some(row) = update.row else { + return Ok(DbtEventStreamUpdate { + status_message: update.status_message, + result: None, + row: None, + finished: false, + }); + }; + let position: DbtRowPosition = DbtRowPosition { + display_index: Some(row.display_index), + display_total: self.display_total, + }; + let detail: String = self.detail(py, &row.result.unique_id)?; + Ok(DbtEventStreamUpdate { + status_message: update.status_message, + result: Some(python_result(py, &row.result)?.unbind()), + row: Some(render_dbt_node_row( + &row.result, + position, + &detail, + self.style, + )), + finished: row.finished, + }) + } +} + +#[pymethods] +impl DbtEventStream { + #[new] + #[pyo3(signature = (**options))] + fn new(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "DbtEventStream", options, STREAM_OPTIONS)?; + Ok(Self { + state: Mutex::new(DbtStreamState::default()), + display_total: arguments.value("display_total")?.extract()?, + detail_by_unique_id: arguments.value("detail_by_unique_id")?.unbind(), + style: CliStyle { + use_color: arguments.flag("use_color")?, + }, + }) + } + + /// The update one output line makes, or `None` when it changes nothing. + fn consume(&self, py: Python<'_>, line: &str) -> PyResult> { + let Some(event) = dbt_event(line)? else { + return Ok(None); + }; + let update: Option = { + let mut state = self.state.lock().map_err(|_| { + PyErr::new::("dbt event stream state is poisoned") + })?; + let (next, update) = consume_dbt_event(std::mem::take(&mut *state), &event); + *state = next; + update + }; + update.map(|update| self.update(py, update)).transpose() + } +} + +/// One dbt node row and its messages. +#[pyfunction] +#[pyo3(signature = (**options))] +fn render_dbt_node_result(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "render_dbt_node_result", options, RENDER_OPTIONS)?; + let position: DbtRowPosition = DbtRowPosition { + display_index: arguments.value("display_index")?.extract()?, + display_total: arguments.value("display_total")?.extract()?, + }; + let detail: String = arguments + .value("detail")? + .extract::>()? + .unwrap_or_default(); + Ok(render_dbt_node_row( + &native_result(&arguments.value("result")?)?, + position, + &detail, + CliStyle { + use_color: arguments.flag("use_color")?, + }, + )) +} + +/// The progress line listing running dbt nodes by their start time. +#[pyfunction] +#[pyo3(signature = (*, active_nodes, now))] +fn format_dbt_active_node_status(active_nodes: Vec<(String, f64)>, now: f64) -> String { + format_dbt_active_nodes(&active_nodes, now) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(render_dbt_node_result, module)?)?; + module.add_function(wrap_pyfunction!(format_dbt_active_node_status, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 09ed288c03..8b309cf090 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -23,6 +23,7 @@ pub(crate) mod cost_report; pub(crate) mod cursor; pub(crate) mod dag_artifact; pub(crate) mod dbt_command; +pub(crate) mod dbt_events; pub(crate) mod dbt_interop; pub(crate) mod debug_command; pub(crate) mod diagnostics; diff --git a/python-boundary.toml b/python-boundary.toml index dbc3ea92c2..d922bf06de 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -174,6 +174,16 @@ path = "integrations/ingestr" reason = "user-code" note = "Loader runtime that runs inside the executor load phase." +[[stays]] +path = "integrations/dbt/_helpers/runtime/event_stream.py" +reason = "user-code" +note = "Runs the dbt subprocess over the user's dbt project and drives its progress threads." + +[[stays]] +path = "integrations/dbt/main/runtime/_execute_json_event_stream.py" +reason = "user-code" +note = "Entry point of the dbt subprocess runner." + [[stays]] path = "compiler/python_nodes/_helpers/identity.py" reason = "user-code" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 55759b3dd8..08c0e5cb6b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -265,6 +265,7 @@ from sqlbuild.integrations.dbt.models import ( DbtLsNode, DbtManifestIndex, DbtManifestModel, + DbtNodeExecutionResult, NormalizedDbtProfileConnection, ResolvedDbtProfileOutput, ) @@ -1593,6 +1594,38 @@ class ConnectionProgressReporter: def on_connection_complete(self, *, connection_count: int, elapsed_seconds: float) -> None: ... def on_connection_error(self, *, connection_count: int, elapsed_seconds: float) -> None: ... +class DbtEventStreamUpdate: + @property + def status_message(self) -> str | None: ... + @property + def result(self) -> DbtNodeExecutionResult | None: ... + @property + def row(self) -> str | None: ... + @property + def finished(self) -> bool: ... + +class DbtEventStream: + def __init__( + self, + *, + display_total: int | None, + detail_by_unique_id: Mapping[str, str] | None, + use_color: bool, + ) -> None: ... + def consume(self, line: str) -> DbtEventStreamUpdate | None: ... + +def render_dbt_node_result( + *, + result: DbtNodeExecutionResult, + use_color: bool, + display_index: int | None = None, + display_total: int | None = None, + detail: str = "", +) -> str: ... +def format_dbt_active_node_status( + *, active_nodes: Sequence[tuple[str, float]], now: float +) -> str: ... + class PlanningProgressReporter: def __init__(self, *, stream: TextIO, use_color: bool = False) -> None: ... def on_progress(self, message: str) -> None: ... diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py index 8c131df15f..84cd7de5a9 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import TextIO, cast -from sqlbuild._native import format_summary_footer +from sqlbuild._native import format_summary_footer, render_dbt_node_result from sqlbuild.integrations.dbt._helpers.graph.core import ( dbt_model_graph_key, expand_combined_downstream, @@ -28,7 +28,6 @@ from sqlbuild.integrations.dbt.main.runtime._execute_json_event_stream import ( execute_dbt_json_event_stream, ) -from sqlbuild.integrations.dbt.main.runtime._render_node_result import render_dbt_node_result from sqlbuild.integrations.dbt.models import ( DbtCliOptions, DbtCombinedGraph, @@ -127,13 +126,15 @@ def execute_dbt_commands( ) streamed_result_count: int = len(results) - len(appended_results) for offset, result in enumerate(appended_results, start=1): - render_dbt_node_result( - stream=stdout_stream, - style=style, - result=result, - display_index=streamed_result_count + offset, - display_total=expected_total if expected_total is not None else len(results), + stdout_stream.write( + render_dbt_node_result( + result=result, + use_color=style.use_color, + display_index=streamed_result_count + offset, + display_total=expected_total if expected_total is not None else len(results), + ) ) + stdout_stream.flush() if on_node_result is not None: on_node_result(result) if on_progress is not None: diff --git a/src/sqlbuild/integrations/dbt/_helpers/runtime/event_stream.py b/src/sqlbuild/integrations/dbt/_helpers/runtime/event_stream.py index ad3096637d..adfd965658 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/runtime/event_stream.py +++ b/src/sqlbuild/integrations/dbt/_helpers/runtime/event_stream.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import os import subprocess import threading @@ -10,67 +9,26 @@ from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path -from typing import TextIO, cast +from typing import TextIO -from sqlbuild._native import format_aligned_name_value +from sqlbuild._native import ( + DbtEventStream, + DbtEventStreamUpdate, + format_dbt_active_node_status, +) from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError -from sqlbuild.integrations.dbt.models import DbtNodeExecutionResult, DbtNodeMessage -from sqlbuild.presentation.classes.cli_style import CliStyle +from sqlbuild.integrations.dbt.models import DbtNodeExecutionResult from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -_DBT_ERROR_LEVEL: str = "error" -_DBT_NODE_FINISHED_EVENT: str = "NodeFinished" -_DBT_NODE_MESSAGE_EVENT_NAMES: frozenset[str] = frozenset( - {"RunResultError", "RunResultFailure", "RunResultWarning", "GenericExceptionOnRun"} -) -_DBT_NODE_MESSAGE_LEVELS: frozenset[str] = frozenset({"warn", "error"}) -_DBT_NODE_STARTED_EVENT_NAMES: frozenset[str] = frozenset({"LogStartLine", "NodeStarted"}) -_DBT_OUTCOME_STATUSES: frozenset[str] = frozenset( - { - "error", - "fail", - "failed", - "ok", - "pass", - "passed", - "skip", - "skipped", - "success", - "warn", - "warning", - } -) -_DBT_RESULT_EVENT_NAMES: frozenset[str] = frozenset( - { - "LogModelResult", - "LogSeedResult", - "LogSnapshotResult", - "LogTestResult", - "LogBatchResult", - "LogFunctionResult", - "NodeFinished", - } -) -_DBT_START_STATUSES: frozenset[str] = frozenset({"start"}) -_DBT_STATUS_ERROR_VALUES: frozenset[str] = frozenset({"error", "fail", "failed"}) -_DBT_STATUS_OK_VALUES: frozenset[str] = frozenset({"ok", "success"}) -_DBT_STATUS_PASS_VALUES: frozenset[str] = frozenset({"pass", "passed"}) -_DBT_STATUS_SKIP_VALUES: frozenset[str] = frozenset({"skip", "skipped"}) -_DBT_STATUS_WARN_VALUES: frozenset[str] = frozenset({"warn", "warning"}) -_DBT_WARN_LEVEL: str = "warn" -_DBT_DURATION_WIDTH: int = 7 _DBT_STATUS_REFRESH_SECONDS: float = 1.0 @dataclass class _DbtStreamState: - pending_messages: dict[str, list[DbtNodeMessage]] = field(default_factory=dict) + events: DbtEventStream results: list[DbtNodeExecutionResult] = field(default_factory=list) - recorded_unique_ids: set[str] = field(default_factory=set) - started_indexes: dict[str, int] = field(default_factory=dict) active_nodes: dict[str, tuple[str, float]] = field(default_factory=dict) - display_index: int = 0 use_color: bool = False @@ -128,8 +86,14 @@ def _execute_dbt_json_event_stream( detail_by_unique_id: dict[str, str] | None, enable_status: bool, ) -> tuple[int, tuple[DbtNodeExecutionResult, ...], int | None]: - style: CliStyle = CliStyle(use_color=use_color) - state: _DbtStreamState = _DbtStreamState(use_color=use_color) + state: _DbtStreamState = _DbtStreamState( + events=DbtEventStream( + display_total=display_total, + detail_by_unique_id=detail_by_unique_id, + use_color=use_color, + ), + use_color=use_color, + ) status_box: dict[str, TransientStatusReporter | None] = {"status": None} status_lock: threading.Lock = threading.Lock() status_stop: threading.Event = threading.Event() @@ -157,12 +121,9 @@ def _execute_dbt_json_event_stream( if process.stdout is not None: try: for line in process.stdout: - state.display_index, status_box = _handle_dbt_output_line( + state, status_box = _handle_dbt_output_line( line=line, stream=stream, - style=style, - display_total=display_total, - detail_by_unique_id=detail_by_unique_id, enable_status=enable_status, state=state, status_box=status_box, @@ -199,100 +160,46 @@ def _handle_dbt_output_line( *, line: str, stream: TextIO, - style: CliStyle, - display_total: int | None, - detail_by_unique_id: dict[str, str] | None, enable_status: bool, state: _DbtStreamState, status_box: dict[str, TransientStatusReporter | None], status_lock: threading.Lock, on_node_result: Callable[[DbtNodeExecutionResult], None] | None, -) -> tuple[int, dict[str, TransientStatusReporter | None]]: - event: dict[str, object] | None = parse_dbt_json_event(line=line) - if event is None: - return state.display_index, status_box - message: DbtNodeMessage | None = parse_dbt_node_message(event=event) - if message is not None: - unique_id: str | None = _event_unique_id(event) - if unique_id is not None: - state.pending_messages.setdefault(unique_id, []).append(message) - return state.display_index, status_box - node_start_message: str | None = parse_dbt_node_start_message(event=event) - if node_start_message is not None: - if enable_status: - with status_lock: - status_box["status"] = _update_message_status( - status=status_box["status"], - message=node_start_message, - stream=stream, - use_color=state.use_color, - ) - start_result: DbtNodeExecutionResult | None = parse_dbt_node_start_result(event=event) - if start_result is not None and start_result.unique_id not in state.started_indexes: - state.display_index += 1 - state.started_indexes[start_result.unique_id] = state.display_index - status_box = _close_status(status_box=status_box, status_lock=status_lock) - render_dbt_node_result( +) -> tuple[_DbtStreamState, dict[str, TransientStatusReporter | None]]: + update: DbtEventStreamUpdate | None = state.events.consume(line) + if update is None: + return state, status_box + if update.status_message is not None and enable_status: + with status_lock: + status_box["status"] = _update_message_status( + status=status_box["status"], + message=update.status_message, stream=stream, - style=style, - result=start_result, - display_index=state.display_index, - display_total=display_total, - detail=_node_detail( - unique_id=start_result.unique_id, - detail_by_unique_id=detail_by_unique_id, - ), + use_color=state.use_color, ) - if enable_status: - with status_lock: - state.active_nodes[start_result.unique_id] = ( - start_result.node_name, - time.monotonic(), - ) - status_box["status"] = _update_active_node_status( - status=status_box["status"], - active_nodes=state.active_nodes, - stream=stream, - use_color=state.use_color, - ) - return state.display_index, status_box - result: DbtNodeExecutionResult | None = parse_dbt_node_result( - event=event, - messages_by_unique_id=state.pending_messages, - ) - if result is None or result.unique_id in state.recorded_unique_ids: - return state.display_index, status_box - state.recorded_unique_ids.add(result.unique_id) + result: DbtNodeExecutionResult | None = update.result + if result is None or update.row is None: + return state, status_box status_box = _close_status(status_box=status_box, status_lock=status_lock) - with status_lock: - state.active_nodes.pop(result.unique_id, None) - state.results.append(result) - if on_node_result is not None: - on_node_result(result) - result_display_index: int | None = state.started_indexes.get(result.unique_id) - if result_display_index is None: - state.display_index += 1 - result_display_index = state.display_index - render_dbt_node_result( - stream=stream, - style=style, - result=result, - display_index=result_display_index, - display_total=display_total, - detail=_node_detail( - unique_id=result.unique_id, - detail_by_unique_id=detail_by_unique_id, - ), - ) + if update.finished: + with status_lock: + state.active_nodes.pop(result.unique_id, None) + state.results.append(result) + if on_node_result is not None: + on_node_result(result) + stream.write(update.row) + stream.flush() if enable_status: with status_lock: + if not update.finished: + state.active_nodes[result.unique_id] = (result.node_name, time.monotonic()) status_box["status"] = _update_active_node_status( status=status_box["status"], active_nodes=state.active_nodes, stream=stream, use_color=state.use_color, ) - return state.display_index, status_box + return state, status_box def _cleanup_dbt_runtime( @@ -475,7 +382,9 @@ def _update_active_node_status( ) -> TransientStatusReporter | None: if not stream.isatty() or not active_nodes: return None - message: str = _format_active_node_status(active_nodes=active_nodes) + message: str = format_dbt_active_node_status( + active_nodes=list(active_nodes.values()), now=time.monotonic() + ) if status is None: status = TransientStatusReporter(stream=stream, use_color=use_color) status.start(message) @@ -484,139 +393,6 @@ def _update_active_node_status( return status -def _format_active_node_status(*, active_nodes: dict[str, tuple[str, float]]) -> str: - now: float = time.monotonic() - ordered: tuple[tuple[str, float], ...] = tuple( - sorted(active_nodes.values(), key=lambda item: item[1]) - ) - displayed: tuple[str, ...] = tuple( - f"{name} {_format_elapsed_seconds(now=now, started_at=started_at)}" - for name, started_at in ordered[:3] - ) - extra_count: int = len(ordered) - len(displayed) - suffix: str = f", +{extra_count} more" if extra_count > 0 else "" - node_label: str = "node" if len(active_nodes) == 1 else "nodes" - return f"running {len(active_nodes)} dbt {node_label}: {', '.join(displayed)}{suffix}" - - -def _format_elapsed_seconds(*, now: float, started_at: float) -> str: - elapsed: int = max(0, int(now - started_at)) - return "<1s" if elapsed == 0 else f"{elapsed}s" - - -def parse_dbt_node_start_message(*, event: dict[str, object]) -> str | None: - """Parse a dbt node-start event into a user-facing progress message.""" - - info: dict[str, object] = _dict_value(event.get("info")) - event_name: str | None = _str_value(info.get("name")) - if event_name not in _DBT_NODE_STARTED_EVENT_NAMES: - return None - data: dict[str, object] = _dict_value(event.get("data")) - node_info: dict[str, object] = _dict_value(data.get("node_info")) - unique_id: str | None = _str_value(node_info.get("unique_id")) - if unique_id is None: - return None - resource_type: str = _str_value(node_info.get("resource_type")) or "node" - node_name: str = _str_value(node_info.get("node_name")) or unique_id - return f"Running dbt {resource_type} {node_name}..." - - -def parse_dbt_node_start_result(*, event: dict[str, object]) -> DbtNodeExecutionResult | None: - """Parse a dbt node-start event into a renderable START row.""" - - info: dict[str, object] = _dict_value(event.get("info")) - event_name: str | None = _str_value(info.get("name")) - if event_name not in _DBT_NODE_STARTED_EVENT_NAMES: - return None - data: dict[str, object] = _dict_value(event.get("data")) - node_info: dict[str, object] = _dict_value(data.get("node_info")) - unique_id: str | None = _str_value(node_info.get("unique_id")) - if unique_id is None: - return None - return DbtNodeExecutionResult( - unique_id=unique_id, - resource_type=_str_value(node_info.get("resource_type")) or "node", - node_name=_str_value(node_info.get("node_name")) or unique_id, - status="start", - index=_int_value(data.get("index")), - total=_int_value(data.get("total")) or _int_value(data.get("num_models")), - execution_time=None, - materialized=_str_value(node_info.get("materialized")), - ) - - -def parse_dbt_json_event(*, line: str) -> dict[str, object] | None: - """Parse one dbt JSON log line, ignoring non-JSON output.""" - - stripped: str = line.strip() - if not stripped or not stripped.startswith("{"): - return None - try: - payload: object = json.loads(stripped) - except json.JSONDecodeError: - return None - return payload if isinstance(payload, dict) else None - - -def parse_dbt_node_message(*, event: dict[str, object]) -> DbtNodeMessage | None: - """Parse a node-scoped warning/error message from a dbt event.""" - - info: dict[str, object] = _dict_value(event.get("info")) - data: dict[str, object] = _dict_value(event.get("data")) - event_name: str | None = _str_value(info.get("name")) - level: str | None = _str_value(info.get("level")) - if event_name not in _DBT_NODE_MESSAGE_EVENT_NAMES and level not in _DBT_NODE_MESSAGE_LEVELS: - return None - if _event_unique_id(event) is None: - return None - message: str | None = _str_value(data.get("msg")) or _str_value(info.get("msg")) - if not message: - return None - return DbtNodeMessage(level=level or "info", message=message.strip()) - - -def parse_dbt_node_result( - *, - event: dict[str, object], - messages_by_unique_id: dict[str, list[DbtNodeMessage]] | None = None, -) -> DbtNodeExecutionResult | None: - """Parse a dbt node final result event.""" - - info: dict[str, object] = _dict_value(event.get("info")) - data: dict[str, object] = _dict_value(event.get("data")) - event_name: str | None = _str_value(info.get("name")) - if event_name not in _DBT_RESULT_EVENT_NAMES: - return None - node_info: dict[str, object] = _dict_value(data.get("node_info")) - unique_id: str | None = _str_value(node_info.get("unique_id")) - if unique_id is None: - return None - if event_name == _DBT_NODE_FINISHED_EVENT: - run_result: dict[str, object] = _dict_value(data.get("run_result")) - status: str | None = _str_value(run_result.get("status")) or _str_value( - node_info.get("node_status") - ) - else: - status = _trusted_status(data.get("status")) or _str_value(node_info.get("node_status")) - relation: dict[str, object] = _dict_value(node_info.get("node_relation")) - messages: tuple[DbtNodeMessage, ...] = tuple((messages_by_unique_id or {}).pop(unique_id, [])) - return DbtNodeExecutionResult( - unique_id=unique_id, - resource_type=_str_value(node_info.get("resource_type")) or "node", - node_name=_str_value(node_info.get("node_name")) or unique_id, - status=status or "unknown", - index=_int_value(data.get("index")), - total=_int_value(data.get("total")) or _int_value(data.get("num_models")), - execution_time=_float_value(data.get("execution_time")), - materialized=_str_value(node_info.get("materialized")), - relation_name=_str_value(relation.get("relation_name")), - database=_str_value(relation.get("database")), - schema=_str_value(relation.get("schema")), - node_checksum=_str_value(node_info.get("node_checksum")), - messages=messages, - ) - - def _build_dbt_json_env(*, target_path: Path | None) -> dict[str, str]: env: dict[str, str] = { **os.environ.copy(), @@ -627,107 +403,3 @@ def _build_dbt_json_env(*, target_path: Path | None) -> dict[str, str]: env["DBT_TARGET_PATH"] = str(target_path) env["DBT_LOG_PATH"] = str(target_path) return env - - -def _event_unique_id(event: dict[str, object]) -> str | None: - data: dict[str, object] = _dict_value(event.get("data")) - node_info: dict[str, object] = _dict_value(data.get("node_info")) - return _str_value(node_info.get("unique_id")) - - -def render_dbt_node_result( - *, - stream: TextIO, - style: CliStyle, - result: DbtNodeExecutionResult, - display_index: int | None = None, - display_total: int | None = None, - detail: str = "", -) -> None: - ctr: str = ( - f"{display_index}/{display_total}" - if display_index is not None and display_total is not None - else f"{result.index}/{result.total}" - if result.index is not None and result.total is not None - else str(display_index) - if display_index is not None - else "-" - ) - resource_type: str = result.resource_type[:9] - name: str = result.node_name[:30] - status: str = _display_status(result.status) - duration: str = f"{result.execution_time:.2f}s" if result.execution_time is not None else "" - rendered_detail: str = style.muted(detail) if detail else "" - value: str = ( - f"{_pad_styled(value=style.status(status=status), plain_value=status, width=6)} " - f"{duration:<{_DBT_DURATION_WIDTH}}{rendered_detail}" - ).rstrip() - stream.write( - f" {ctr:<5} {resource_type:<9}" - + format_aligned_name_value( - plain_name=name, - styled_name=style.dbt_object_name(name), - value=value, - name_column_width=30, - prefix=" ", - ) - + "\n" - ) - for message in result.messages: - message_status: str = ( - _DBT_WARN_LEVEL if message.level == _DBT_WARN_LEVEL else _DBT_ERROR_LEVEL - ) - stream.write(f" {style.status(status=message_status):<9} {message.message}\n") - stream.flush() - - -def _node_detail(*, unique_id: str, detail_by_unique_id: dict[str, str] | None) -> str: - if detail_by_unique_id is None: - return "" - return detail_by_unique_id.get(unique_id, "") - - -def _display_status(status: str) -> str: - normalized: str = status.lower() - if normalized in _DBT_STATUS_OK_VALUES: - return "OK" - if normalized in _DBT_STATUS_PASS_VALUES: - return "PASS" - if normalized in _DBT_STATUS_WARN_VALUES: - return "WARN" - if normalized in _DBT_STATUS_SKIP_VALUES: - return "SKIP" - if normalized in _DBT_STATUS_ERROR_VALUES: - return "FAIL" - if normalized in _DBT_START_STATUSES: - return "START" - return status.upper() - - -def _pad_styled(*, value: str, plain_value: str, width: int) -> str: - return value + " " * max(0, width - len(plain_value)) - - -def _trusted_status(value: object | None) -> str | None: - status: str | None = _str_value(value) - if status is None: - return None - return status if status.lower() in _DBT_OUTCOME_STATUSES else None - - -def _dict_value(value: object | None) -> dict[str, object]: - return cast(dict[str, object], value) if isinstance(value, dict) else {} - - -def _str_value(value: object | None) -> str | None: - return value if isinstance(value, str) and value else None - - -def _int_value(value: object | None) -> int | None: - return value if isinstance(value, int) else None - - -def _float_value(value: object | None) -> float | None: - if isinstance(value, int | float): - return float(value) - return None diff --git a/src/sqlbuild/integrations/dbt/main/runtime/_render_node_result.py b/src/sqlbuild/integrations/dbt/main/runtime/_render_node_result.py deleted file mode 100644 index ccaf5522a3..0000000000 --- a/src/sqlbuild/integrations/dbt/main/runtime/_render_node_result.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Render a dbt node result.""" - -from typing import TextIO - -from sqlbuild.integrations.dbt._helpers.runtime.event_stream import ( - render_dbt_node_result as _render, -) -from sqlbuild.integrations.dbt.models import DbtNodeExecutionResult -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def render_dbt_node_result( - *, - stream: TextIO, - style: CliStyle, - result: DbtNodeExecutionResult, - display_index: int | None = None, - display_total: int | None = None, - detail: str = "", -) -> None: - """Render one dbt node execution result.""" - - _ = _render( - stream=stream, - style=style, - result=result, - display_index=display_index, - display_total=display_total, - detail=detail, - ) diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index 51b29f713e..e8f31ae397 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -7,7 +7,12 @@ from types import MappingProxyType from typing import cast -from sqlbuild._native import assemble_project, extract_sql_references +from sqlbuild._native import ( + DbtEventStream, + DbtEventStreamUpdate, + assemble_project, + extract_sql_references, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, @@ -497,3 +502,15 @@ def graph_key_from_stable_id(stable_id: str) -> DbtCombinedGraphKey: ): sqlbuild_model_graph_key, } return factories[(owner_enum, resource_type_enum)](name) + + +def consume_dbt_events(*events: dict[str, object]) -> DbtEventStreamUpdate | None: + """Feed dbt events to a fresh native stream and return the last update.""" + + stream: DbtEventStream = DbtEventStream( + display_total=None, detail_by_unique_id=None, use_color=False + ) + updates: tuple[DbtEventStreamUpdate | None, ...] = tuple( + stream.consume(json.dumps(event)) for event in events + ) + return updates[-1] diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_event_stream.py b/tests/unit/src/sqlbuild/integrations/dbt/test_event_stream.py index 7485e3be9a..dc27cf2af1 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_event_stream.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_event_stream.py @@ -1,6 +1,7 @@ from __future__ import annotations import io +import json import threading import time from collections.abc import Callable, Iterator @@ -8,15 +9,11 @@ import pytest +from sqlbuild._native import DbtEventStream, DbtEventStreamUpdate from sqlbuild.integrations.dbt._helpers.runtime.event_stream import ( execute_dbt_json_event_stream, - parse_dbt_json_event, - parse_dbt_node_message, - parse_dbt_node_result, - parse_dbt_node_start_message, - parse_dbt_node_start_result, ) -from sqlbuild.integrations.dbt.models import DbtNodeExecutionResult, DbtNodeMessage +from sqlbuild.integrations.dbt.models import DbtNodeExecutionResult from sqlbuild.observability import ( EventDispatcher, LifecycleEvent, @@ -29,6 +26,7 @@ DbtRuntimeCleanupTestCase, DbtSilentStatusRefreshTestCase, ) +from tests.unit.src.sqlbuild.integrations.dbt.helpers import consume_dbt_events _DBT_RESULT_LINE: str = ( '{"data":{"execution_time":1.0,"index":1,"total":1,"status":"success",' @@ -173,12 +171,12 @@ def test_given_dbt_result_event_when_parsing_then_returns_node_execution_result( test_case: DbtEventParseTestCase, ) -> None: - result: DbtNodeExecutionResult | None = parse_dbt_node_result( - event=test_case.event, - messages_by_unique_id=None, - ) + update: DbtEventStreamUpdate | None = consume_dbt_events(test_case.event) + assert update is not None + result: DbtNodeExecutionResult | None = update.result assert result is not None + assert update.finished is True assert result.unique_id == test_case.expected_unique_id assert result.resource_type == test_case.expected_resource_type assert result.node_name == test_case.expected_node_name @@ -207,7 +205,7 @@ def test_given_dbt_result_event_when_parsing_then_returns_node_execution_result( "unique_id": "model.demo.fact_orders", }, }, - "info": {"level": "error", "name": "LogModelResult", "msg": "ERROR"}, + "info": {"level": "info", "name": "LogModelResult", "msg": "ERROR"}, }, expected_unique_id="model.demo.fact_orders", expected_resource_type="model", @@ -221,21 +219,30 @@ def test_given_dbt_result_event_when_parsing_then_returns_node_execution_result( def test_given_buffered_node_message_when_parsing_result_then_attaches_message( test_case: DbtEventParseTestCase, ) -> None: - messages_by_unique_id: dict[str, list[DbtNodeMessage]] = { - test_case.expected_unique_id: [ - DbtNodeMessage(level="error", message="relation raw.orders does not exist") - ] + message_event: dict[str, object] = { + "data": { + "msg": " relation raw.orders does not exist\n", + "node_info": {"unique_id": test_case.expected_unique_id}, + }, + "info": {"level": "error", "name": "RunResultError"}, } - - result: DbtNodeExecutionResult | None = parse_dbt_node_result( - event=test_case.event, - messages_by_unique_id=messages_by_unique_id, + stream: DbtEventStream = DbtEventStream( + display_total=None, detail_by_unique_id=None, use_color=False ) + message_update: DbtEventStreamUpdate | None = stream.consume(json.dumps(message_event)) + update: DbtEventStreamUpdate | None = stream.consume(json.dumps(test_case.event)) + repeated: DbtEventStreamUpdate | None = stream.consume(json.dumps(test_case.event)) + + assert message_update is None + assert repeated is None + assert update is not None + result: DbtNodeExecutionResult | None = update.result assert result is not None assert len(result.messages) == test_case.expected_message_count assert result.messages[0].message == "relation raw.orders does not exist" - assert messages_by_unique_id == {} + assert update.row is not None + assert "error relation raw.orders does not exist" in update.row @pytest.mark.parametrize( @@ -266,14 +273,21 @@ def test_given_buffered_node_message_when_parsing_result_then_attaches_message( ], ids=lambda case: case.description, ) -def test_given_node_scoped_error_event_when_parsing_then_returns_node_message( +def test_given_node_scoped_error_event_when_parsing_then_attaches_node_message( test_case: DbtEventParseTestCase, ) -> None: - message: DbtNodeMessage | None = parse_dbt_node_message(event=test_case.event) + result_event: dict[str, object] = { + "data": {"status": "error", "node_info": {"unique_id": test_case.expected_unique_id}}, + "info": {"level": "info", "name": "LogModelResult"}, + } + + update: DbtEventStreamUpdate | None = consume_dbt_events(test_case.event, result_event) - assert message is not None - assert message.level == test_case.expected_status - assert "Database Error" in message.message + assert update is not None + result: DbtNodeExecutionResult | None = update.result + assert result is not None + assert result.messages[0].level == test_case.expected_status + assert "Database Error" in result.messages[0].message @pytest.mark.parametrize( @@ -296,14 +310,12 @@ def test_given_node_scoped_error_event_when_parsing_then_returns_node_message( def test_given_non_result_event_when_parsing_then_returns_none( test_case: DbtEventParseTestCase, ) -> None: - non_json_event: dict[str, object] | None = parse_dbt_json_event(line="not json") - result: DbtNodeExecutionResult | None = parse_dbt_node_result( - event=test_case.event, - messages_by_unique_id=None, + stream: DbtEventStream = DbtEventStream( + display_total=None, detail_by_unique_id=None, use_color=False ) - assert non_json_event is None - assert result is None + assert stream.consume("not json") is None + assert consume_dbt_events(test_case.event) is None assert test_case.expected_unique_id == "" assert test_case.expected_status == "" @@ -839,7 +851,7 @@ def kill(self) -> None: lambda *args, **kwargs: FailingThread(), ) monkeypatch.setattr( - "sqlbuild.integrations.dbt._helpers.runtime.event_stream.parse_dbt_json_event", + "sqlbuild.integrations.dbt._helpers.runtime.event_stream._handle_dbt_output_line", lambda **kwargs: (_ for _ in ()).throw(RuntimeError(test_case.expected_error)), ) @@ -975,9 +987,10 @@ def wait(self) -> int: def test_given_node_started_event_when_parsing_then_returns_progress_message( test_case: DbtEventParseTestCase, ) -> None: - message: str | None = parse_dbt_node_start_message(event=test_case.event) + update: DbtEventStreamUpdate | None = consume_dbt_events(test_case.event) - assert message == test_case.expected_status + assert update is not None + assert update.status_message == test_case.expected_status @pytest.mark.parametrize( @@ -1009,9 +1022,12 @@ def test_given_node_started_event_when_parsing_then_returns_progress_message( def test_given_node_started_event_when_parsing_then_returns_start_result( test_case: DbtEventParseTestCase, ) -> None: - result: DbtNodeExecutionResult | None = parse_dbt_node_start_result(event=test_case.event) + update: DbtEventStreamUpdate | None = consume_dbt_events(test_case.event) + assert update is not None + result: DbtNodeExecutionResult | None = update.result assert result is not None + assert update.finished is False assert result.unique_id == test_case.expected_unique_id assert result.resource_type == test_case.expected_resource_type assert result.node_name == test_case.expected_node_name From 4b9b3e87a86d9f0a6a2c21c9edf7895c3cdb979a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 01:14:32 +0100 Subject: [PATCH 12/24] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index d922bf06de..4c4548bab2 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 67089 +remaining_lines = 66290 [[stays]] path = "executor" From 704dbac46de0af5946a08dd59062678b8ee0d7b2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 01:31:45 +0100 Subject: [PATCH 13/24] test: parse the dbt interop project before the native defer-to setup build --- .../sqlbuild/cli/commands/main/dbt/helpers.py | 20 +++++++++++++++++++ .../cli/commands/main/dbt/test_execute.py | 2 ++ 2 files changed, 22 insertions(+) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py index e79a4f0c04..762d53e2ff 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py @@ -103,6 +103,26 @@ def prepare_dbt_interop_project(*, tmp_path: Path) -> Path: return root_dir / "sqlbuild_project" +def write_dbt_interop_manifest(*, project_dir: Path) -> None: + """Parse the interop fixture's dbt project so SQLBuild can resolve its dbt refs.""" + + dbt_project_dir: Path = project_dir.parent / "dbt_project" + subprocess.run( + ( + dbt_executable(), + "parse", + "--project-dir", + str(dbt_project_dir), + "--profiles-dir", + str(project_dir.parent / "profiles"), + ), + cwd=dbt_project_dir, + capture_output=True, + check=True, + text=True, + ) + + def write_sqlbuild_defer_target_models(*, project_dir: Path) -> None: """Configure native SQLBuild target deferral inside a dbt interop fixture.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py index a2a29bb6a5..c364a9b16d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py @@ -19,6 +19,7 @@ load_json_stdout, prepare_dbt_interop_project, skip_unless_dbt_is_runnable, + write_dbt_interop_manifest, write_sqlbuild_defer_target_models, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( @@ -373,6 +374,7 @@ def test_given_native_defer_to_when_running_dbt_interop_then_reads_deferred_targ skip_unless_dbt_is_runnable() project_dir: Path = prepare_dbt_interop_project(tmp_path=tmp_path) write_sqlbuild_defer_target_models(project_dir=project_dir) + write_dbt_interop_manifest(project_dir=project_dir) local_config_path: Path = project_dir / "sqlbuild_local.toml" local_config_path.write_text('target = "prod"\n', encoding="utf-8") setup_result: subprocess.CompletedProcess[str] = run_sqb( From 318ab223df2be2b27eb28a07024b4c1dfb000ddd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 01:33:35 +0100 Subject: [PATCH 14/24] refactor(discovery): resolve named declaration roles and external model descriptions natively --- .../_helpers/discovery/description_inputs.rs | 82 +++++++++++++++++++ .../src/bindings/_helpers/discovery/mod.rs | 1 + .../_helpers/discovery/registration.rs | 9 +- .../bindings/_helpers/format_command/run.rs | 15 ++-- .../src/bindings/_helpers/rules/lint_text.rs | 18 ++++ .../sqlbuild-rules/src/lint_text/constants.rs | 20 +++++ .../sqlbuild-rules/src/lint_text/main/mod.rs | 1 + .../main/named_declaration_role_kind.rs | 35 ++++++++ .../lint_text/tests/test_format_settings.rs | 42 ++++++++++ .../src/lint_text/tests/test_types.rs | 6 ++ python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 4 +- .../classes/model_description_resolution.py | 27 ------ .../main/_model_description_inputs.py | 40 --------- .../main/named_declaration_role_kind.py | 30 ------- src/sqlbuild/lint/_helpers/headers.py | 7 +- .../sqlbuild/compiler/discovery/helpers.py | 4 +- 17 files changed, 224 insertions(+), 119 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs create mode 100644 crates/sqlbuild-rules/src/lint_text/main/named_declaration_role_kind.rs delete mode 100644 src/sqlbuild/compiler/compile/classes/model_description_resolution.py delete mode 100644 src/sqlbuild/compiler/discovery/main/_model_description_inputs.py delete mode 100644 src/sqlbuild/compiler/discovery/main/named_declaration_role_kind.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs new file mode 100644 index 0000000000..300f6fc990 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs @@ -0,0 +1,82 @@ +//! Model description resolution for header-only tooling, without importing project Python. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyCFunction, PyFrozenSet}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; + +/// Models and model schemas, skipping files that cannot be read. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +pub(crate) fn discover_model_description_inputs<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let native = |name: &str| imported(py, NATIVE_MODULE, name); + let project_config: Bound<'py, PyAny> = + native("load_project_config_file")?.call1((project_dir,))?; + let skip: Bound<'py, PyAny> = + PyCFunction::new_closure(py, None, None, |_arguments, _keywords| -> PyResult<()> { + Ok(()) + })? + .into_any(); + let disabled: Bound<'py, PyAny> = PyBool::new(py, false).to_owned().into_any(); + let scope: Bound<'py, PyAny> = call_with( + &native("directory_snapshot_scope")?, + &[("project_dir", project_dir.clone())], + )?; + within(&scope, |_| { + call_with( + &imported(py, DISCOVERY_MODELS_MODULE, "DiscoveredProjectInputs")?, + &[ + ("project_config", project_config), + ( + "local_config", + imported(py, CONTRACT_MODELS_MODULE, "LocalConfig")?.call0()?, + ), + ( + "model_files", + call_with( + &native("discover_model_files")?, + &[ + ("project_dir", project_dir.clone()), + ("extract_implicit_alias_columns", disabled.clone()), + ("extract_output_column_locations", disabled), + ("on_fault", skip.clone()), + ], + )?, + ), + ( + "model_schema_files", + call_with( + &native("discover_model_schema_files")?, + &[("project_dir", project_dir.clone()), ("on_fault", skip)], + )?, + ), + ], + ) + }) +} + +/// Model files that `[defaults]`, `[path_defaults]` or a model schema describe. +pub(crate) fn externally_described_models<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let inputs: Bound<'py, PyAny> = discover_model_description_inputs(project_dir)?; + let schemas: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "public_model_schema_index")?.call1((&inputs,))?; + let described: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "externally_described_models")?.call1((&inputs, schemas))?; + PyFrozenSet::new(py, described.try_iter()?.collect::>>()?) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(discover_model_description_inputs, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index c0a9bd0a15..991c53a503 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod configuration; pub(crate) mod declaration_files; pub(crate) mod declaration_records; +pub(crate) mod description_inputs; pub(crate) mod faults; pub(crate) mod guidance; pub(crate) mod header_declarations; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs index b2817f23b9..93148c6a47 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs @@ -3,10 +3,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::discovery::{ - declaration_files, declaration_records, guidance, header_declarations, header_values, - hook_entries, key_checks, locations, model_records, project_files, project_validation, - python_function_files, python_hook_files, python_module_files, raised, scope_snapshot, - snapshot, sql_test_records, yaml_entries, yaml_records, + declaration_files, declaration_records, description_inputs, guidance, header_declarations, + header_values, hook_entries, key_checks, locations, model_records, project_files, + project_validation, python_function_files, python_hook_files, python_module_files, raised, + scope_snapshot, snapshot, sql_test_records, yaml_entries, yaml_records, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -29,6 +29,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { yaml_entries::register(module)?; header_values::register(module)?; guidance::register(module)?; + description_inputs::register(module)?; hook_entries::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs index 82ca20dad6..5b9d3a77aa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs @@ -8,6 +8,7 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PyTuple} use sqlbuild_core::format_command::main::collect_lint_files::collect_lint_files; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::discovery::description_inputs::externally_described_models; use crate::bindings::_helpers::format_command::files::{ FormatChange, file_dictionary, overlaid, persist_changes, sorted_items, }; @@ -15,7 +16,6 @@ use crate::bindings::_helpers::format_command::files::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const FIXES_MODULE: &str = "sqlbuild.lint._helpers.fixes"; const FIXTURE_AUTOFIX_MODULE: &str = "sqlbuild.compiler.planner.classes.fixture_null_autofix"; -const DESCRIPTION_MODULE: &str = "sqlbuild.compiler.compile.classes.model_description_resolution"; const LINT_ERRORS_MODULE: &str = "sqlbuild.lint.exceptions"; const DESCRIPTION_PRESENT: &str = "description-present"; @@ -198,15 +198,10 @@ fn drop_resolved_description_faults<'py>( if !has_description_fault { return Ok(violations); } - let py: Python<'py> = project_dir.py(); - let described: BTreeSet = call_with( - &imported(py, DESCRIPTION_MODULE, "ModelDescriptionResolution")? - .getattr("externally_described_models")?, - &[("project_dir", project_dir.clone())], - )? - .try_iter()? - .map(|path| path?.call_method0("resolve")?.extract()) - .collect::>()?; + let described: BTreeSet = externally_described_models(project_dir)? + .try_iter()? + .map(|path| path?.call_method0("resolve")?.extract()) + .collect::>()?; let mut kept: Vec> = Vec::new(); for violation in violations { let resolved: PathBuf = violation diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs index e637d4bf30..db2392bac6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs @@ -11,6 +11,7 @@ use sqlbuild_rules::lint_text::errors::LintTextError; use sqlbuild_rules::lint_text::main::apply_suppressions::apply_suppressions; use sqlbuild_rules::lint_text::main::format_sql_bodies::format_sql_bodies; use sqlbuild_rules::lint_text::main::interpolation_text_at::interpolation_text_at; +use sqlbuild_rules::lint_text::main::named_declaration_role_kind::named_declaration_role_kind as role_kind; use sqlbuild_rules::lint_text::main::neutralize_audit_parameters::neutralize_audit_parameters; use sqlbuild_rules::lint_text::main::neutralize_context_interpolation::neutralize_context_interpolation; use sqlbuild_rules::lint_text::main::neutralize_interpolation::neutralize_interpolation; @@ -27,6 +28,7 @@ use crate::bindings::types::CompilerDetach; const LINT_MODELS_MODULE: &str = "sqlbuild.lint.models"; const LINT_TYPES_MODULE: &str = "sqlbuild.lint.types"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const NATIVE_ENGINE: &str = "native"; const FORMAT_UNSAFE_CODE: &str = "format-unsafe"; const FORMAT_UNSAFE_REMEDIATION: &str = "Rewrite the unsupported SQL shape or report a minimal formatter reproduction. No header, SQL body, or fixture changes were written to this file."; @@ -119,6 +121,21 @@ fn scan_fixture_typed_null_candidates<'py>( )) } +/// The named declaration kind of a project-relative path, as `DeclarationKind`. +#[pyfunction] +#[pyo3(signature = (*, relative_path))] +fn named_declaration_role_kind<'py>( + relative_path: &Bound<'py, PyAny>, +) -> PyResult>> { + let parts: Vec = relative_path.getattr("parts")?.extract()?; + match role_kind(&parts) { + Some(kind) => Ok(Some( + imported(relative_path.py(), SCOPE_TYPES_MODULE, "DeclarationKind")?.call1((kind,))?, + )), + None => Ok(None), + } +} + /// Remove violations matched by suppressions and add the suppression findings, as lint models. #[pyfunction] fn apply_lint_suppressions<'py>( @@ -327,5 +344,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module )?)?; module.add_function(wrap_pyfunction!(format_lint_sql_bodies, module)?)?; + module.add_function(wrap_pyfunction!(named_declaration_role_kind, module)?)?; module.add_function(wrap_pyfunction!(apply_lint_suppressions, module)?) } diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index 28b1785577..b713e61363 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -141,3 +141,23 @@ pub(crate) const SCOPED_DECLARATION_DIRECTORIES: [&str; 6] = [ "_enums", "_constants", ]; +/// `GLOBAL_NAMED_DECLARATION_ROOTS`, in Python's dict order, with their declaration kinds. +pub(crate) const GLOBAL_NAMED_ROOT_KINDS: [(&[&str], &str); 5] = [ + (&["audits", "generic"], "audit"), + (&["audits", "singular"], "singular_audit"), + (&["schemas"], "schema"), + (&["hooks", "sql"], "sql_hook"), + (&["hooks", "python"], "python_hook"), +]; +/// `GROUPED_NAMED_DECLARATION_ROLES`, in Python's dict order, with their declaration kinds. +pub(crate) const GROUPED_NAMED_ROLE_KINDS: [(&[&str], &str); 9] = [ + (&["audits", "generic"], "audit"), + (&["_audits", "generic"], "audit"), + (&["audits", "singular"], "singular_audit"), + (&["schemas"], "schema"), + (&["_schemas"], "schema"), + (&["hooks", "sql"], "sql_hook"), + (&["_hooks", "sql"], "sql_hook"), + (&["hooks", "python"], "python_hook"), + (&["_hooks", "python"], "python_hook"), +]; diff --git a/crates/sqlbuild-rules/src/lint_text/main/mod.rs b/crates/sqlbuild-rules/src/lint_text/main/mod.rs index 1c2440670a..9b6cd55683 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/mod.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/mod.rs @@ -3,6 +3,7 @@ pub mod first_header_only; pub mod format_sql_bodies; pub mod interpolation_text_at; pub mod lint_sql_bodies; +pub mod named_declaration_role_kind; pub mod neutralize_audit_parameters; pub mod neutralize_context_interpolation; pub mod neutralize_interpolation; diff --git a/crates/sqlbuild-rules/src/lint_text/main/named_declaration_role_kind.rs b/crates/sqlbuild-rules/src/lint_text/main/named_declaration_role_kind.rs new file mode 100644 index 0000000000..4347c0451e --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/named_declaration_role_kind.rs @@ -0,0 +1,35 @@ +//! The named declaration kind a project path's folders give a file. + +use crate::lint_text::constants::{ + DECLARATION_GROUP_DIRECTORY, GLOBAL_NAMED_ROOT_KINDS, GROUPED_NAMED_ROLE_KINDS, +}; + +fn starts_with(parts: &[String], prefix: &[&str]) -> bool { + parts.len() >= prefix.len() && parts.iter().zip(prefix).all(|(part, role)| part == role) +} + +/// The `DeclarationKind` value of a global named root or a role under `_sqlbuild`, if any. +#[must_use] +pub fn named_declaration_role_kind(relative_parts: &[String]) -> Option<&'static str> { + let folders: &[String] = relative_parts + .split_last() + .map_or(&[], |(_, folders)| folders); + if let Some((_, kind)) = GLOBAL_NAMED_ROOT_KINDS + .iter() + .find(|(root, _)| starts_with(folders, root)) + { + return Some(kind); + } + for (index, part) in folders.iter().enumerate() { + if part != DECLARATION_GROUP_DIRECTORY { + continue; + } + if let Some((_, kind)) = GROUPED_NAMED_ROLE_KINDS + .iter() + .find(|(role, _)| starts_with(&folders[index + 1..], role)) + { + return Some(kind); + } + } + None +} diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_format_settings.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_format_settings.rs index 95603e8bbf..bb7dd8c193 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_format_settings.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_format_settings.rs @@ -102,3 +102,45 @@ fn given_relative_paths_when_classifying_headers_then_only_plain_models_scan_one ); } } + +#[test] +fn given_project_paths_when_classifying_named_roles_then_kinds_match_python() { + let test_cases = [ + crate::lint_text::tests::test_types::RoleKindTestCase { + description: "a generic audit", + relative_parts: &["audits", "generic", "not_null.sql"], + expected_kind: Some("audit"), + }, + crate::lint_text::tests::test_types::RoleKindTestCase { + description: "a local SQL hook in a declaration group", + relative_parts: &["models", "orders", "_sqlbuild", "_hooks", "sql", "h.sql"], + expected_kind: Some("sql_hook"), + }, + crate::lint_text::tests::test_types::RoleKindTestCase { + description: "a file named like a root is not under it", + relative_parts: &["schemas.sql"], + expected_kind: None, + }, + crate::lint_text::tests::test_types::RoleKindTestCase { + description: "a plain model", + relative_parts: &["models", "orders.sql"], + expected_kind: None, + }, + ]; + + for test_case in test_cases { + let parts: Vec = test_case + .relative_parts + .iter() + .map(|part| (*part).to_owned()) + .collect(); + assert_eq!( + crate::lint_text::main::named_declaration_role_kind::named_declaration_role_kind( + &parts + ), + test_case.expected_kind, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs index 372cfcd932..8dde7c9547 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs @@ -67,3 +67,9 @@ pub(super) struct FirstHeaderTestCase { pub(super) relative_parts: &'static [&'static str], pub(super) expected_first_only: bool, } + +pub(super) struct RoleKindTestCase { + pub(super) description: &'static str, + pub(super) relative_parts: &'static [&'static str], + pub(super) expected_kind: Option<&'static str>, +} diff --git a/python-boundary.toml b/python-boundary.toml index 4c4548bab2..bb22807743 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 66290 +remaining_lines = 66194 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 08c0e5cb6b..4048a0c04e 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -221,7 +221,7 @@ from sqlbuild.compiler.scopes.models import ( UsageRecord, VisibilityResolution, ) -from sqlbuild.compiler.scopes.types import JsonValue, ScopeQueryTarget +from sqlbuild.compiler.scopes.types import DeclarationKind, JsonValue, ScopeQueryTarget from sqlbuild.compiler.source_freshness.models import SourceFreshnessIdentity, SourceFreshnessRecord from sqlbuild.compiler.source_freshness.types import ( SourceFreshnessAgeStatus, @@ -3016,6 +3016,8 @@ def serialize_hook_entries( sql_fields: tuple[str, ...], python_hook_fields: Mapping[str, Mapping[str, object]] | None = None, ) -> list[dict[str, object]]: ... +def named_declaration_role_kind(*, relative_path: PurePath) -> DeclarationKind | None: ... +def discover_model_description_inputs(*, project_dir: Path) -> DiscoveredProjectInputs: ... def microbatch_guidance() -> tuple[str, str]: ... def explicit_references_help(*, allowed: str) -> str: ... def run_format_command( diff --git a/src/sqlbuild/compiler/compile/classes/model_description_resolution.py b/src/sqlbuild/compiler/compile/classes/model_description_resolution.py deleted file mode 100644 index b0d21a08e3..0000000000 --- a/src/sqlbuild/compiler/compile/classes/model_description_resolution.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Compile-equivalent description resolution for header-only tooling.""" - -from __future__ import annotations - -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery.main._model_description_inputs import ( - discover_model_description_inputs, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, ModelSchemaDeclaration - - -class ModelDescriptionResolution: - """Resolve model descriptions that come from outside the MODEL header.""" - - @staticmethod - def externally_described_models(*, project_dir: Path) -> frozenset[Path]: - """Return model files that `[defaults]`, `[path_defaults]` or a model schema describe.""" - - discovered_inputs: DiscoveredProjectInputs = discover_model_description_inputs( - project_dir=project_dir - ) - model_schemas: dict[str, ModelSchemaDeclaration] = _native.public_model_schema_index( - discovered_inputs - ) - return frozenset(_native.externally_described_models(discovered_inputs, model_schemas)) diff --git a/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py b/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py deleted file mode 100644 index bd2649e7d0..0000000000 --- a/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py +++ /dev/null @@ -1,40 +0,0 @@ -"""SQL-only discovery of the inputs that decide a model's description.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild._native import ( - directory_snapshot_scope, - discover_model_files, - discover_model_schema_files, - load_project_config_file, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryFileFault -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -def discover_model_description_inputs(*, project_dir: Path) -> DiscoveredProjectInputs: - """Load project config, model files and model schemas without importing project Python.""" - - project_config: ProjectConfig = load_project_config_file(project_dir) - with directory_snapshot_scope(project_dir=project_dir): - return DiscoveredProjectInputs( - project_config=project_config, - local_config=LocalConfig(), - model_files=discover_model_files( - project_dir=project_dir, - extract_implicit_alias_columns=False, - extract_output_column_locations=False, - on_fault=_skip_unreadable_file, - ), - model_schema_files=discover_model_schema_files( - project_dir=project_dir, on_fault=_skip_unreadable_file - ), - ) - - -def _skip_unreadable_file(fault: DiscoveryFileFault) -> None: - """Leave an unparseable file unresolved; format reports that file's own faults.""" - - del fault diff --git a/src/sqlbuild/compiler/discovery/main/named_declaration_role_kind.py b/src/sqlbuild/compiler/discovery/main/named_declaration_role_kind.py deleted file mode 100644 index 85572f9e26..0000000000 --- a/src/sqlbuild/compiler/discovery/main/named_declaration_role_kind.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Classify an authored path by the named declaration role that contains it.""" - -from __future__ import annotations - -from pathlib import PurePath - -from sqlbuild.compiler.scopes.constants import ( - DECLARATION_GROUP_DIRECTORY, - GLOBAL_NAMED_DECLARATION_ROOTS, - GROUPED_NAMED_DECLARATION_ROLES, -) -from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind - - -def named_declaration_role_kind(*, relative_path: PurePath) -> DeclarationKind | None: - """Return the audit, schema, or hook kind for a project-relative declaration file path.""" - - parts: tuple[str, ...] = relative_path.parts[:-1] - for role_parts, kind in GLOBAL_NAMED_DECLARATION_ROOTS.items(): - if parts[: len(role_parts)] == role_parts: - return kind - for index, part in enumerate(parts): - if part != DECLARATION_GROUP_DIRECTORY: - continue - grouped_role: tuple[str, ...] - role: tuple[DeclarationKind, ScopeKind] - for grouped_role, role in GROUPED_NAMED_DECLARATION_ROLES.items(): - if parts[index + 1 : index + 1 + len(grouped_role)] == grouped_role: - return role[0] - return None diff --git a/src/sqlbuild/lint/_helpers/headers.py b/src/sqlbuild/lint/_helpers/headers.py index b98fdfeed0..b9eeb3232b 100644 --- a/src/sqlbuild/lint/_helpers/headers.py +++ b/src/sqlbuild/lint/_helpers/headers.py @@ -4,11 +4,12 @@ from pathlib import Path, PurePath -from sqlbuild._native import measurement_body_ranges, sql_body_ranges -from sqlbuild.compiler.compile.constants import HOOK_DIRECTORY_NAME -from sqlbuild.compiler.discovery.main.named_declaration_role_kind import ( +from sqlbuild._native import ( + measurement_body_ranges, named_declaration_role_kind, + sql_body_ranges, ) +from sqlbuild.compiler.compile.constants import HOOK_DIRECTORY_NAME from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.lint.models import HeaderSpan, LintFileRole diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index e272e693ff..1c873934ed 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -24,6 +24,7 @@ discover_audit_files, discover_constant_files, discover_enum_files, + discover_model_description_inputs, discover_model_schema_files, discover_project_inputs, discover_scope_snapshot, @@ -36,9 +37,6 @@ SQL_FUNCTION_HEADER_KEYS, SQL_HOOK_HEADER_KEYS, ) -from sqlbuild.compiler.discovery.main._model_description_inputs import ( - discover_model_description_inputs, -) from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, DiscoveredSourceFile, From 87b24d429a5faaedc5524679bd3ddf27c6971f2f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 01:33:58 +0100 Subject: [PATCH 15/24] chore: keep the user Python resource diagnostics reader as user-code --- python-boundary.toml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index bb22807743..5428df2b72 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 66194 +remaining_lines = 66074 [[stays]] path = "executor" @@ -204,6 +204,11 @@ path = "compiler/discovery/main/runtime_extensions.py" reason = "user-code" note = "Binds imported project providers to event exporter and sink declarations." +[[stays]] +path = "compiler/compile/_helpers/diagnostics/python_resources.py" +reason = "user-code" +note = "Reads docstrings and source lines of loaded project Python functions and provider classes." + [[stays]] path = "compiler/discovery/_helpers/filesystem/command_output_sinks.py" reason = "user-code" From 16bcb64030048b6eac01b6aed9a6c4efa9bfd6ef Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:13:49 +0100 Subject: [PATCH 16/24] refactor: resolve Python node selectors and graph indexes natively Port Python-node, unified SQL/Python run and check-exclude selector resolution plus the Python node graph edge, tag and path indexes to sqlbuild-analysis. The node inventory that reads loaded user functions moves to compiler/python_nodes/_helpers/user_inventory.py (user-code). --- crates/sqlbuild-analysis/src/lib.rs | 1 + .../python_selection/_helpers/inventory.rs | 85 +++ .../src/python_selection/_helpers/mod.rs | 4 + .../src/python_selection/_helpers/nodes.rs | 287 ++++++++ .../src/python_selection/_helpers/unified.rs | 442 ++++++++++++ .../python_selection/_helpers/validation.rs | 205 ++++++ .../src/python_selection/constants.rs | 39 + .../src/python_selection/main/mod.rs | 4 + .../main/python_node_graph_indexes.rs | 6 + .../main/resolve_python_node_excludes.rs | 23 + .../main/resolve_python_node_selectors.rs | 14 + .../main/resolve_python_sql_run_selection.rs | 21 + .../src/python_selection/mod.rs | 9 + .../src/python_selection/models.rs | 89 +++ .../src/python_selection/tests.rs | 6 + .../src/python_selection/tests/helpers.rs | 198 +++++ .../tests/test_python_selection.rs | 255 +++++++ .../src/python_selection/tests/test_types.rs | 28 + .../bindings/_helpers/check/check_command.rs | 15 +- .../src/bindings/_helpers/graph/mod.rs | 1 + .../_helpers/graph/python_selection.rs | 217 ++++++ .../bindings/_helpers/graph/registration.rs | 1 + python-boundary.toml | 7 +- src/sqlbuild/_native.pyi | 22 + .../cli/commands/_helpers/check/core.py | 30 +- .../python_nodes/_helpers/run_selectors.py | 81 --- .../python_nodes/_helpers/selectors.py | 177 ----- .../_helpers/unified_selectors.py | 675 ------------------ .../{inventory.py => user_inventory.py} | 98 +-- .../compiler/python_nodes/constants.py | 9 - .../compiler/python_nodes/main/_graph.py | 2 +- .../python_nodes/main/_run_selection.py | 19 +- .../compiler/python_nodes/main/selectors.py | 14 - .../compiler/python_nodes/_helpers/helpers.py | 20 +- .../python_nodes/_helpers/test_inventory.py | 2 +- .../_helpers/test_run_selectors.py | 6 +- .../python_nodes/_helpers/test_selectors.py | 6 +- .../_helpers/test_unified_selectors.py | 39 +- .../executor/python_nodes/_helpers/helpers.py | 2 +- 39 files changed, 2047 insertions(+), 1112 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/constants.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/main/python_node_graph_indexes.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/main/resolve_python_node_excludes.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/main/resolve_python_node_selectors.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/mod.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/models.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/tests.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs delete mode 100644 src/sqlbuild/compiler/python_nodes/_helpers/run_selectors.py delete mode 100644 src/sqlbuild/compiler/python_nodes/_helpers/selectors.py delete mode 100644 src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py rename src/sqlbuild/compiler/python_nodes/_helpers/{inventory.py => user_inventory.py} (78%) delete mode 100644 src/sqlbuild/compiler/python_nodes/constants.py delete mode 100644 src/sqlbuild/compiler/python_nodes/main/selectors.py diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 2168789c56..fc478207ea 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -20,6 +20,7 @@ pub mod lineage_command; pub mod lineage_graph; pub mod manifest; pub mod python_plan; +pub mod python_selection; pub mod python_values; pub mod query_analysis; pub mod relation_names; diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs new file mode 100644 index 0000000000..9a416f0c44 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs @@ -0,0 +1,85 @@ +//! Python's Python-node graph inventory: dependency edges and the tag and path indexes. + +use std::collections::HashMap; + +use crate::python_selection::constants::{CURRENT_DIRECTORY, ROOT_DIRECTORY}; +use crate::python_selection::models::{InventoryIndexes, InventoryNode}; + +/// Rows keyed like a Python dict: first insertion keeps its position. +#[derive(Default)] +struct OrderedRows { + rows: Vec<(String, V)>, + positions: HashMap, +} + +impl OrderedRows { + fn entry(&mut self, key: &str) -> &mut V { + let position: usize = match self.positions.get(key) { + Some(&position) => position, + None => { + self.positions.insert(key.to_owned(), self.rows.len()); + self.rows.push((key.to_owned(), V::default())); + self.rows.len() - 1 + } + }; + &mut self.rows[position].1 + } +} + +/// `PurePosixPath(relative_path).parent.as_posix()` for a normalised POSIX path. +fn parent_folder(relative_path: &str) -> String { + match relative_path.rsplit_once('/') { + Some(("", _)) => ROOT_DIRECTORY.to_owned(), + Some((parent, _)) => parent.to_owned(), + None => CURRENT_DIRECTORY.to_owned(), + } +} + +fn adjacency( + nodes: &[InventoryNode], + edges: &[(String, String)], + reversed: bool, +) -> Vec<(String, Vec)> { + let mut rows: OrderedRows> = OrderedRows::default(); + for node in nodes { + rows.entry(&node.name); + } + for (upstream, downstream) in edges { + let (from, to) = if reversed { + (upstream, downstream) + } else { + (downstream, upstream) + }; + rows.entry(from).push(to.clone()); + rows.entry(to); + } + rows.rows +} + +/// The edges, upstream and downstream adjacency, tag index and path index of the nodes. +pub(crate) fn indexes(nodes: &[InventoryNode]) -> InventoryIndexes { + let mut edges: Vec<(String, String)> = Vec::new(); + for node in nodes { + for upstream in &node.upstream { + edges.push((upstream.clone(), node.name.clone())); + } + } + let mut tags: OrderedRows> = OrderedRows::default(); + let mut paths: OrderedRows = OrderedRows::default(); + for node in nodes { + for tag in &node.tags { + let names: &mut Vec = tags.entry(tag); + if !names.contains(&node.name) { + names.push(node.name.clone()); + } + } + *paths.entry(&node.name) = parent_folder(&node.relative_path); + } + InventoryIndexes { + upstream: adjacency(nodes, &edges, false), + downstream: adjacency(nodes, &edges, true), + edges, + tags: tags.rows, + paths: paths.rows, + } +} diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/mod.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/mod.rs new file mode 100644 index 0000000000..a4363db587 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod inventory; +pub(crate) mod nodes; +pub(crate) mod unified; +pub(crate) mod validation; diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs new file mode 100644 index 0000000000..40a7abb782 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs @@ -0,0 +1,287 @@ +//! Python's `resolve_python_node_selectors`: selector resolution over Python node names. + +use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; + +use sqlbuild_core::text::main::glob_match::glob_match; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::main::parse_selector::parse_selector; +use crate::graph::models::{EdgeDirection, ParsedSelector, ProjectGraph}; +use crate::python_selection::constants::{ + ASSET_KIND, CHECK_KIND, INTERSECTION_SEPARATOR, LOADER_KIND, NAME_KIND, PATH_BETWEEN_CODE, + PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PATTERN_CHARACTERS, PYTHON_NODE_PATH_ROOT, + SQL_MODEL_PATH_ROOT, TAG_KIND, TASK_KIND, TEST_KIND, UNIT_TEST_CODE, + UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_NAME_CODE, UNKNOWN_PATH_CODE, UNKNOWN_TAG_CODE, + UNMAPPED_KIND_CODE, +}; +use crate::python_selection::models::{NodeIndex, PythonSelectionGraph, PythonSelectionNode}; + +pub(crate) type Names = BTreeSet; +type Resolved = Result; + +pub(crate) fn error(code: &'static str, message: String) -> SelectorError { + SelectorError { + code, + message, + help: None, + } +} + +/// The error for a unit-test selector where unit tests never run. +pub(crate) fn unit_test_error(raw: &str) -> SelectorError { + error( + UNIT_TEST_CODE, + format!("selector '{raw}' selects a unit test; {UNIT_TEST_ONLY_TEST_AND_BUILD}"), + ) +} + +pub(crate) fn is_pattern(value: &str) -> bool { + value.contains(PATTERN_CHARACTERS) +} + +/// A path selector value as a folder: forward slashes, no leading or trailing slash. +pub(crate) fn selector_folder(value: &str) -> String { + value.replace('\\', "/").trim_matches('/').to_owned() +} + +/// The first segment of a selector folder. +pub(crate) fn folder_root(folder: &str) -> &str { + folder.split('/').next().unwrap_or_default() +} + +fn keyed(rows: &[(String, Vec)]) -> HashMap<&str, &[String]> { + rows.iter() + .map(|(name, values)| (name.as_str(), values.as_slice())) + .collect() +} + +impl<'graph> NodeIndex<'graph> { + pub(crate) fn new(graph: &'graph PythonSelectionGraph) -> Self { + Self { + graph, + by_name: graph + .nodes + .iter() + .map(|node| (node.name.as_str(), node)) + .collect(), + upstream: keyed(&graph.upstream), + downstream: keyed(&graph.downstream), + tags: keyed(&graph.tags), + } + } + + pub(crate) fn node(&self, name: &str) -> Option<&'graph PythonSelectionNode> { + self.by_name.get(name).copied() + } + + pub(crate) fn edges(&self, name: &str, direction: EdgeDirection) -> &'graph [String] { + let edges: &HashMap<&'graph str, &'graph [String]> = match direction { + EdgeDirection::Upstream => &self.upstream, + EdgeDirection::Downstream => &self.downstream, + }; + edges.get(name).copied().unwrap_or_default() + } + + /// Every name reachable from `starts`; a start itself only through a cycle. + pub(crate) fn closure<'name>( + &self, + starts: impl IntoIterator, + direction: EdgeDirection, + ) -> Names { + let mut visited: HashSet<&str> = HashSet::new(); + let mut frontier: VecDeque<&str> = VecDeque::new(); + for start in starts { + frontier.push_back(start); + } + while let Some(current) = frontier.pop_front() { + for neighbor in self.edges(current, direction) { + if visited.insert(neighbor) { + frontier.push_back(neighbor); + } + } + } + visited.into_iter().map(str::to_owned).collect() + } + + /// Whether `name` is a node of `kind`. + pub(crate) fn is_kind(&self, name: &str, kind: &str) -> bool { + self.node(name).is_some_and(|node| node.kind == kind) + } +} + +/// `select` minus `exclude` over Python nodes; every node when nothing is selected. +pub(crate) fn select_nodes( + index: &NodeIndex<'_>, + select: &[String], + exclude: &[String], +) -> Resolved { + if select.is_empty() { + return Ok(index + .by_name + .keys() + .map(|name| (*name).to_owned()) + .collect()); + } + let selected: Names = groups(index, select)?; + let excluded: Names = groups(index, exclude)?; + Ok(selected.difference(&excluded).cloned().collect()) +} + +fn groups(index: &NodeIndex<'_>, raw_groups: &[String]) -> Resolved { + let mut resolved: Names = Names::new(); + for group in raw_groups { + for token in group.split_whitespace() { + resolved.extend(intersection(index, token)?); + } + } + Ok(resolved) +} + +fn intersection(index: &NodeIndex<'_>, token: &str) -> Resolved { + let parts: Vec = token + .split(INTERSECTION_SEPARATOR) + .map(|part| single(index, part)) + .collect::>()?; + let mut parts = parts.into_iter(); + let mut result: Names = parts.next().unwrap_or_default(); + for part in parts { + result = result.intersection(&part).cloned().collect(); + } + Ok(result) +} + +fn single(index: &NodeIndex<'_>, raw: &str) -> Resolved { + let ParsedSelector::Kind { + kind, + value, + upstream, + downstream, + } = parse_selector(raw)? + else { + return Err(error( + PATH_BETWEEN_CODE, + format!("path-between selector '{raw}' is not supported for Python nodes yet"), + )); + }; + let names: Names = match kind.as_str() { + TEST_KIND => return Err(unit_test_error(raw)), + TAG_KIND => tagged(index, &value)?, + PATH_KIND => under_path(index, &value)?, + _ => { + let names: Names = lookup(index, &kind, &value)?; + if names.is_empty() { + return Err(error( + UNKNOWN_NAME_CODE, + format!("unknown Python node selector '{value}'"), + )); + } + names + } + }; + Ok(expanded(index, names, (upstream, downstream))) +} + +fn tagged(index: &NodeIndex<'_>, tag: &str) -> Resolved { + let names: Names = index + .tags + .get(tag) + .copied() + .unwrap_or_default() + .iter() + .cloned() + .collect(); + if names.is_empty() { + return Err(error( + UNKNOWN_TAG_CODE, + format!("no Python nodes found with tag '{tag}'"), + )); + } + Ok(names) +} + +fn under_path(index: &NodeIndex<'_>, value: &str) -> Resolved { + let folder: String = selector_folder(value); + let root: &str = folder_root(&folder); + if root != PYTHON_NODE_PATH_ROOT && root != SQL_MODEL_PATH_ROOT { + return Err(error(PATH_ROOT_CODE, PATH_ROOT_ERROR.to_owned())); + } + let nested: String = format!("{folder}/"); + let names: Names = index + .graph + .paths + .iter() + .filter(|(_, indexed)| { + folder.is_empty() || *indexed == folder || indexed.starts_with(&nested) + }) + .map(|(name, _)| name.clone()) + .collect(); + if names.is_empty() { + return Err(error( + UNKNOWN_PATH_CODE, + format!("no Python nodes found under path '{folder}'"), + )); + } + Ok(names) +} + +fn lookup(index: &NodeIndex<'_>, kind: &str, value: &str) -> Resolved { + if kind == NAME_KIND { + if !is_pattern(value) { + return Ok(index + .node(value) + .map(|node| node.name.clone()) + .into_iter() + .collect()); + } + return Ok(index + .by_name + .iter() + .filter(|(name, _)| glob_match(name, value)) + .map(|(_, node)| node.name.clone()) + .collect()); + } + if ![TASK_KIND, ASSET_KIND, LOADER_KIND, CHECK_KIND].contains(&kind) { + return Err(error( + UNMAPPED_KIND_CODE, + format!("selector type '{kind}' does not map to a Python node type"), + )); + } + if !is_pattern(value) { + return Ok(index + .graph + .nodes + .iter() + .rev() + .find(|node| node.kind == kind && node.name == value) + .map(|node| node.name.clone()) + .into_iter() + .collect()); + } + Ok(index + .by_name + .values() + .filter(|node| node.kind == kind && glob_match(&node.name, value)) + .map(|node| node.name.clone()) + .collect()) +} + +fn expanded(index: &NodeIndex<'_>, names: Names, (upstream, downstream): (bool, bool)) -> Names { + let mut result: Names = names.clone(); + if upstream { + result.extend(index.closure(names.iter().map(String::as_str), EdgeDirection::Upstream)); + } + if downstream { + result.extend(index.closure(names.iter().map(String::as_str), EdgeDirection::Downstream)); + } + result +} + +/// Python's `all_keys`: selector name to SQL key. +pub(crate) fn sql_names(graph: &ProjectGraph) -> HashMap<&str, &ObjectKey> { + graph + .names + .iter() + .map(|(name, key)| (name.as_str(), key)) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs new file mode 100644 index 0000000000..6357ed80b9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -0,0 +1,442 @@ +//! Python's unified selector resolution across SQL resources and Python nodes. + +use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; + +use sqlbuild_core::text::main::glob_match::glob_match; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::main::build_resources::build_resources; +use crate::graph::main::edge_keys::edge_keys; +use crate::graph::main::parse_selector::parse_selector; +use crate::graph::main::resolve_selectors::resolve_selectors; +use crate::graph::models::{BuildResources, EdgeDirection, ParsedSelector}; +use crate::python_selection::_helpers::nodes::{ + Names, error, folder_root, is_pattern, select_nodes, selector_folder, unit_test_error, +}; +use crate::python_selection::_helpers::validation::{ + source_loaders, terminal_loaders, validate_boundaries, validate_selected_dependencies, +}; +use crate::python_selection::constants::{ + ASSET_KIND, CHECK_KIND, EXPANSION_MARKER, INTERSECTION_SEPARATOR, LOADER_KIND, NAME_KIND, + PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PLANNER_DEFAULT_CODE, PYTHON_NODE_PATH_ROOT, + SEED_KIND, SOURCE_KIND, SOURCE_RESOURCE, SQL_MODEL_PATH_ROOT, TAG_KIND, TASK_KIND, TEST_KIND, + UNKNOWN_NAME_CODE, UNKNOWN_TAG_CODE, +}; +use crate::python_selection::models::{Atom, PythonSelectionNode, RunSelection, SelectionContext}; + +type Atoms = BTreeSet; +type Resolved = Result; + +fn python_atoms(names: Names) -> Atoms { + names.into_iter().map(Atom::Python).collect() +} + +fn sql_atoms(keys: impl IntoIterator) -> Atoms { + keys.into_iter().map(Atom::Sql).collect() +} + +fn sql_keys(atoms: &Atoms) -> Vec { + atoms + .iter() + .filter_map(|atom| match atom { + Atom::Sql(key) => Some(key.clone()), + Atom::Python(_) => None, + }) + .collect() +} + +/// Python's `resolve_python_sql_run_selectors`. +pub(crate) fn run_selection( + context: &SelectionContext<'_>, + select: &[String], + exclude: &[String], +) -> Result { + validate_boundaries(context)?; + let mut selected: Atoms = if select.is_empty() { + let mut every: Atoms = + sql_atoms(context.sql.graph.names.iter().map(|(_, key)| key.clone())); + every.extend( + context + .nodes + .by_name + .keys() + .map(|name| Atom::Python((*name).to_owned())), + ); + every + } else { + groups(context, select)? + }; + let excluded: Atoms = groups(context, exclude)?; + selected = selected.difference(&excluded).cloned().collect(); + let required: Atoms = sql_atoms(build_resources( + context.sql.graph, + &sql_keys(&selected), + BuildResources::SELECTION, + )); + selected.extend(required); + let is_check = + |atom: &Atom| matches!(atom, Atom::Python(name) if context.nodes.is_kind(name, CHECK_KIND)); + let checks: Vec<&str> = selected + .iter() + .filter(|atom| is_check(atom)) + .filter_map(|atom| match atom { + Atom::Python(name) => Some(name.as_str()), + Atom::Sql(_) => None, + }) + .collect(); + if !select.is_empty() && !checks.is_empty() { + return Err(error( + PLANNER_DEFAULT_CODE, + format!( + "Python checks are not selectable here: {}. Use sqb check instead.", + checks.join(", ") + ), + )); + } + let runnable: Atoms = selected + .into_iter() + .filter(|atom| !is_check(atom)) + .collect(); + validate_selected_dependencies(context, &runnable)?; + let terminal: HashSet<&str> = context + .sql + .sources + .iter() + .filter_map(|(_, loader)| loader.as_deref()) + .collect(); + let mut keys: BTreeSet = sql_keys(&runnable).into_iter().collect(); + let mut names: Vec = Vec::new(); + for atom in &runnable { + let Atom::Python(name) = atom else { + continue; + }; + if context.nodes.is_kind(name, LOADER_KIND) && !terminal.contains(name.as_str()) { + keys.insert((SOURCE_RESOURCE.to_owned(), name.clone())); + } + names.push(name.clone()); + } + Ok(RunSelection { + sql_keys: keys.into_iter().collect(), + python_node_names: names, + }) +} + +fn groups(context: &SelectionContext<'_>, raw_groups: &[String]) -> Resolved { + let mut resolved: Atoms = Atoms::new(); + for group in raw_groups { + for token in group.split_whitespace() { + resolved.extend(intersection(context, token)?); + } + } + Ok(resolved) +} + +fn intersection(context: &SelectionContext<'_>, token: &str) -> Resolved { + let parts: Vec = token + .split(INTERSECTION_SEPARATOR) + .map(|part| single(context, part)) + .collect::>()?; + let mut parts = parts.into_iter(); + let mut result: Atoms = parts.next().unwrap_or_default(); + for part in parts { + result = result.intersection(&part).cloned().collect(); + } + Ok(result) +} + +fn single(context: &SelectionContext<'_>, raw: &str) -> Resolved { + let (upstream, downstream) = match parse_selector(raw)? { + ParsedSelector::Path { .. } => return single_side(context, raw), + ParsedSelector::Kind { + upstream, + downstream, + .. + } => (upstream, downstream), + }; + if !upstream && !downstream { + return single_side(context, raw); + } + let core: &str = raw + .trim() + .trim_start_matches(EXPANSION_MARKER) + .trim_end_matches(EXPANSION_MARKER); + let mut atoms: Atoms = Atoms::new(); + if upstream { + let resolved: Atoms = single_side(context, &format!("{EXPANSION_MARKER}{core}"))?; + let dependencies: Atoms = sql_dependency_upstream(context, &resolved); + atoms.extend(resolved); + atoms.extend(dependencies); + } + if downstream { + let resolved: Atoms = single_side(context, &format!("{core}{EXPANSION_MARKER}"))?; + let dependents: Atoms = sql_dependent_python(context, &resolved); + atoms.extend(resolved); + atoms.extend(dependents); + } + Ok(atoms) +} + +/// The SQL resources selected Python nodes declare, with their SQL upstream and loaders. +fn sql_dependency_upstream(context: &SelectionContext<'_>, atoms: &Atoms) -> Atoms { + let starts: BTreeSet<&ObjectKey> = atoms + .iter() + .filter_map(|atom| match atom { + Atom::Python(name) => context.nodes.node(name), + Atom::Sql(_) => None, + }) + .flat_map(|node| node.sql_deps.iter()) + .collect(); + if starts.is_empty() { + return Atoms::new(); + } + let mut visited: HashSet<&ObjectKey> = starts.iter().copied().collect(); + let mut frontier: VecDeque<&ObjectKey> = starts.iter().copied().collect(); + while let Some(current) = frontier.pop_front() { + for neighbor in edge_keys(context.sql.graph, current, EdgeDirection::Upstream) { + if visited.insert(neighbor) { + frontier.push_back(neighbor); + } + } + } + let selected: Atoms = sql_atoms(visited.into_iter().cloned()); + let loaders: Atoms = terminal_loader_atoms(context, &selected); + selected.into_iter().chain(loaders).collect() +} + +fn declares_any(node: &PythonSelectionNode, keys: &HashSet<&ObjectKey>) -> bool { + node.sql_deps.iter().any(|key| keys.contains(key)) +} + +/// Runnable Python nodes declaring a selected SQL resource, with their downstream nodes. +fn sql_dependent_python(context: &SelectionContext<'_>, atoms: &Atoms) -> Atoms { + let selected: HashSet<&ObjectKey> = atoms + .iter() + .filter_map(|atom| match atom { + Atom::Sql(key) => Some(key), + Atom::Python(_) => None, + }) + .collect(); + let dependents: BTreeSet<&str> = context + .nodes + .graph + .nodes + .iter() + .filter(|node| node.kind != CHECK_KIND) + .filter(|node| declares_any(node, &selected)) + .map(|node| node.name.as_str()) + .collect(); + if dependents.is_empty() { + return Atoms::new(); + } + let mut names: Names = context + .nodes + .closure(dependents.iter().copied(), EdgeDirection::Downstream); + names.extend(dependents.iter().map(|name| (*name).to_owned())); + python_atoms( + names + .into_iter() + .filter(|name| !context.nodes.is_kind(name, CHECK_KIND)) + .collect(), + ) +} + +fn single_side(context: &SelectionContext<'_>, raw: &str) -> Resolved { + let (kind, value, upstream) = match parse_selector(raw)? { + ParsedSelector::Path { upstream, .. } => { + return Ok(with_loaders(context, resolve_sql(context, raw)?, upstream)); + } + ParsedSelector::Kind { + kind, + value, + upstream, + .. + } => (kind, value, upstream), + }; + let atoms: Atoms = match kind.as_str() { + TEST_KIND => return Err(unit_test_error(raw)), + PATH_KIND => resolve_path(context, raw, &value)?, + SEED_KIND | SOURCE_KIND => resolve_sql(context, raw)?, + TASK_KIND | ASSET_KIND | LOADER_KIND | CHECK_KIND => { + return resolve_typed(context, raw, (&kind, &value)); + } + TAG_KIND => resolve_tag(context, raw, &value)?, + NAME_KIND => resolve_name(context, raw, &value)?, + _ => { + return Err(error( + PLANNER_DEFAULT_CODE, + format!("unsupported selector '{raw}'"), + )); + } + }; + Ok(with_loaders(context, atoms, upstream)) +} + +fn with_loaders(context: &SelectionContext<'_>, atoms: Atoms, upstream: bool) -> Atoms { + if !upstream { + return atoms; + } + let loaders: Atoms = terminal_loader_atoms(context, &atoms); + atoms.into_iter().chain(loaders).collect() +} + +fn resolve_sql(context: &SelectionContext<'_>, raw: &str) -> Resolved { + Ok(sql_atoms(resolve_selectors( + context.sql.graph, + &[raw.to_owned()], + &[], + )?)) +} + +fn resolve_python(context: &SelectionContext<'_>, raw: &str) -> Resolved { + Ok(python_atoms(select_nodes( + &context.nodes, + &[raw.to_owned()], + &[], + )?)) +} + +fn resolve_typed( + context: &SelectionContext<'_>, + raw: &str, + (kind, value): (&str, &str), +) -> Resolved { + if kind == LOADER_KIND { + let terminal: HashMap<&str, &str> = terminal_loaders(context.sql); + let matched: Option<&&str> = terminal.keys().filter(|name| glob_match(name, value)).min(); + if let Some(&loader) = matched { + return Err(error( + UNKNOWN_NAME_CODE, + format!( + "loader selector '{value}' matches managed source loader '{loader}'; \ + select it as source:{}", + terminal[loader] + ), + )); + } + } + resolve_python(context, raw) +} + +fn resolve_tag(context: &SelectionContext<'_>, raw: &str, tag: &str) -> Resolved { + let mut atoms: Atoms = Atoms::new(); + for resolved in [resolve_sql(context, raw), resolve_python(context, raw)] { + match resolved { + Ok(found) => atoms.extend(found), + Err(failure) if failure.code == UNKNOWN_TAG_CODE => {} + Err(failure) => return Err(failure), + } + } + if atoms.is_empty() { + return Err(error( + PLANNER_DEFAULT_CODE, + format!("no SQL resources or Python nodes found with tag '{tag}'"), + )); + } + Ok(atoms) +} + +fn resolve_path(context: &SelectionContext<'_>, raw: &str, value: &str) -> Resolved { + let folder: String = selector_folder(value); + match folder_root(&folder) { + SQL_MODEL_PATH_ROOT => resolve_sql(context, raw), + PYTHON_NODE_PATH_ROOT => resolve_python(context, raw), + _ => Err(error(PATH_ROOT_CODE, PATH_ROOT_ERROR.to_owned())), + } +} + +fn both_error(name: &str) -> SelectorError { + error( + PLANNER_DEFAULT_CODE, + format!( + "selector name '{name}' matches both a SQL resource and a Python node; \ + resource names must be globally unique" + ), + ) +} + +fn unknown_name(value: &str) -> SelectorError { + error( + UNKNOWN_NAME_CODE, + format!("unknown selector name '{value}'"), + ) +} + +fn resolve_name(context: &SelectionContext<'_>, raw: &str, value: &str) -> Resolved { + let terminal: HashMap<&str, &str> = terminal_loaders(context.sql); + if !is_pattern(value) { + let sql: bool = context.sql_names.contains_key(value); + let python: bool = context.nodes.node(value).is_some(); + return match (sql, python) { + (true, _) if terminal.contains_key(value) => resolve_sql(context, raw), + (true, true) => Err(both_error(value)), + (true, false) => resolve_sql(context, raw), + (false, true) => resolve_python(context, raw), + (false, false) => Err(unknown_name(value)), + }; + } + let sql_names: BTreeSet<&str> = context + .sql + .graph + .names + .iter() + .map(|(name, _)| name.as_str()) + .filter(|name| glob_match(name, value)) + .collect(); + let python_names: BTreeSet<&str> = context + .nodes + .by_name + .keys() + .copied() + .filter(|name| glob_match(name, value)) + .collect(); + if let Some(conflict) = sql_names + .intersection(&python_names) + .find(|name| !terminal.contains_key(*name)) + { + return Err(both_error(conflict)); + } + let mut atoms: Atoms = Atoms::new(); + if !sql_names.is_empty() { + atoms.extend(resolve_sql(context, raw)?); + } + if !python_names.is_empty() { + let duplicates: HashSet<&str> = sql_names + .iter() + .copied() + .filter(|name| terminal.contains_key(name)) + .collect(); + atoms.extend(resolve_python(context, raw)?.into_iter().filter( + |atom| !matches!(atom, Atom::Python(name) if duplicates.contains(name.as_str())), + )); + } + if atoms.is_empty() { + return Err(unknown_name(value)); + } + Ok(atoms) +} + +/// Python's `_required_terminal_loader_atoms`: loaders of selected sources and their upstream. +fn terminal_loader_atoms(context: &SelectionContext<'_>, atoms: &Atoms) -> Atoms { + let loaders: HashMap<&str, &str> = source_loaders(context.sql).into_iter().collect(); + let mut required: Atoms = Atoms::new(); + for atom in atoms { + let Atom::Sql((resource, source)) = atom else { + continue; + }; + if resource != SOURCE_RESOURCE { + continue; + } + let Some(&loader) = loaders.get(source.as_str()) else { + continue; + }; + if context.nodes.node(loader).is_none() { + continue; + } + required.insert(Atom::Python(loader.to_owned())); + required.extend(python_atoms( + context.nodes.closure([loader], EdgeDirection::Upstream), + )); + } + required +} diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs new file mode 100644 index 0000000000..ddacd27837 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs @@ -0,0 +1,205 @@ +//! Python's SQL/Python graph boundary rules and selected-dependency checks. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use crate::graph::errors::SelectorError; +use crate::graph::models::EdgeDirection; +use crate::python_selection::_helpers::nodes::error; +use crate::python_selection::constants::{ + ASSET_KIND, CHECK_KIND, LOADER_KIND, MODEL_RESOURCE, PLANNER_DEFAULT_CODE, SOURCE_RESOURCE, + SQL_MODEL_DEPENDENCY_TYPES, TASK_KIND, +}; +use crate::python_selection::models::{Atom, SelectionContext, SqlSelectionGraph}; + +fn failure(message: String) -> SelectorError { + error(PLANNER_DEFAULT_CODE, message) +} + +/// `{source name: loader}` for sources with a loader, as Python's dict builds it. +pub(crate) fn source_loaders<'graph>( + sql: SqlSelectionGraph<'graph>, +) -> Vec<(&'graph str, &'graph str)> { + let mut rows: Vec<(&str, &str)> = Vec::new(); + let mut positions: HashMap<&str, usize> = HashMap::new(); + for (source, loader) in sql.sources { + let Some(loader) = loader else { + continue; + }; + match positions.get(source.as_str()) { + Some(&position) => rows[position].1 = loader, + None => { + positions.insert(source, rows.len()); + rows.push((source, loader)); + } + } + } + rows +} + +/// `{loader: source}` over [`source_loaders`], a later source winning a shared loader. +pub(crate) fn terminal_loaders<'graph>( + sql: SqlSelectionGraph<'graph>, +) -> HashMap<&'graph str, &'graph str> { + source_loaders(sql) + .into_iter() + .map(|(source, loader)| (loader, source)) + .collect() +} + +/// Python's `validate_python_sql_boundaries`. +pub(crate) fn validate_boundaries(context: &SelectionContext<'_>) -> Result<(), SelectorError> { + validate_sql_model_dependencies(context)?; + validate_python_sql_refs(context)?; + validate_loader_upstreams(context)?; + let terminal: HashMap<&str, &str> = terminal_loaders(context.sql); + for node in &context.nodes.graph.nodes { + for dependency in context.nodes.edges(&node.name, EdgeDirection::Upstream) { + let Some(source) = terminal.get(dependency.as_str()) else { + continue; + }; + if node.kind == CHECK_KIND { + return Err(failure(format!( + "Check '{}' depends on terminal loader '{dependency}'; \ + use source audits for source '{source}' instead", + node.name + ))); + } + if node.kind == TASK_KIND || node.kind == ASSET_KIND { + return Err(failure(format!( + "Python node '{}' depends on terminal loader '{dependency}'; \ + depend on source '{source}' instead", + node.name + ))); + } + } + } + Ok(()) +} + +fn validate_sql_model_dependencies(context: &SelectionContext<'_>) -> Result<(), SelectorError> { + let sources: HashSet<&str> = context + .sql + .sources + .iter() + .map(|(name, _)| name.as_str()) + .collect(); + for (model, dependencies) in &context.sql.graph.upstream { + if model.0 != MODEL_RESOURCE { + continue; + } + for dependency in dependencies { + if !SQL_MODEL_DEPENDENCY_TYPES.contains(&dependency.0.as_str()) { + return Err(failure(format!( + "SQL model '{}' depends on non-SQL resource '{}:{}'; \ + SQL model dependencies must stay SQL-only", + model.1, dependency.0, dependency.1 + ))); + } + if dependency.0 == SOURCE_RESOURCE && !sources.contains(dependency.1.as_str()) { + return Err(failure(format!( + "SQL model '{}' depends on intermediate loader '{}'; \ + depend on a source populated by a terminal loader instead", + model.1, dependency.1 + ))); + } + } + } + Ok(()) +} + +fn validate_python_sql_refs(context: &SelectionContext<'_>) -> Result<(), SelectorError> { + for node in &context.nodes.graph.nodes { + for (kind, name) in &node.sql_deps { + let Some(key) = context.sql_names.get(name.as_str()) else { + return Err(failure(format!( + "Python node '{}' depends on unknown SQL resource '{name}'", + node.name + ))); + }; + if &key.0 != kind { + return Err(failure(format!( + "Python node '{}' declares {kind}('{name}') but '{name}' is a {}", + node.name, key.0 + ))); + } + } + } + Ok(()) +} + +fn validate_loader_upstreams(context: &SelectionContext<'_>) -> Result<(), SelectorError> { + for loader in context + .nodes + .graph + .nodes + .iter() + .filter(|node| node.kind == LOADER_KIND) + { + for upstream_name in context + .nodes + .closure([loader.name.as_str()], EdgeDirection::Upstream) + { + let Some(upstream) = context.nodes.node(&upstream_name) else { + continue; + }; + if (upstream.kind == TASK_KIND || upstream.kind == ASSET_KIND) + && !upstream.sql_deps.is_empty() + { + return Err(failure(format!( + "Loader '{}' depends on Python node '{}' which depends on SQL; \ + Python-to-SQL writes must flow through a pre-SQL task/asset -> loader -> \ + source path", + loader.name, upstream.name + ))); + } + } + } + Ok(()) +} + +/// Python's `_validate_selected_dependencies`: selected sources and nodes have their loaders. +pub(crate) fn validate_selected_dependencies( + context: &SelectionContext<'_>, + atoms: &BTreeSet, +) -> Result<(), SelectorError> { + let selected: HashSet<&str> = atoms + .iter() + .filter_map(|atom| match atom { + Atom::Python(name) => Some(name.as_str()), + Atom::Sql(_) => None, + }) + .collect(); + let loaders: HashMap<&str, &str> = source_loaders(context.sql).into_iter().collect(); + for atom in atoms { + let Atom::Sql((resource, source)) = atom else { + continue; + }; + if resource != SOURCE_RESOURCE { + continue; + } + let Some(&loader) = loaders.get(source.as_str()) else { + continue; + }; + if loader == source || selected.contains(loader) { + continue; + } + return Err(failure(format!( + "Source '{source}' requires loader '{loader}', but that loader was not selected; \ + select it directly or use upstream expansion" + ))); + } + for atom in atoms { + let Atom::Python(name) = atom else { + continue; + }; + for upstream in context.nodes.edges(name, EdgeDirection::Upstream) { + if !selected.contains(upstream.as_str()) { + return Err(failure(format!( + "Python node '{name}' depends on unselected Python node '{upstream}'; \ + select it directly or use upstream expansion" + ))); + } + } + } + Ok(()) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/constants.rs b/crates/sqlbuild-analysis/src/python_selection/constants.rs new file mode 100644 index 0000000000..3bd1d1c28a --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/constants.rs @@ -0,0 +1,39 @@ +//! Python node kinds, selector kinds, error codes and messages of Python's node selectors. + +pub(crate) const TASK_KIND: &str = "task"; +pub(crate) const ASSET_KIND: &str = "asset"; +pub(crate) const LOADER_KIND: &str = "loader"; +pub(crate) const CHECK_KIND: &str = "check"; +pub(crate) const NAME_KIND: &str = "name"; +pub(crate) const TAG_KIND: &str = "tag"; +pub(crate) const PATH_KIND: &str = "path"; +pub(crate) const TEST_KIND: &str = "test"; +pub(crate) const SEED_KIND: &str = "seed"; +pub(crate) const SOURCE_KIND: &str = "source"; +pub(crate) const MODEL_RESOURCE: &str = "model"; +pub(crate) const SOURCE_RESOURCE: &str = "source"; +/// Resource types a SQL model may depend on. +pub(crate) const SQL_MODEL_DEPENDENCY_TYPES: [&str; 6] = + ["model", "source", "seed", "udf", "table_fn", "dbt_ref"]; +pub(crate) const EXPANSION_MARKER: char = '+'; +pub(crate) const INTERSECTION_SEPARATOR: char = ','; +pub(crate) const PATTERN_CHARACTERS: [char; 3] = ['*', '?', '[']; +pub(crate) const SQL_MODEL_PATH_ROOT: &str = "models"; +pub(crate) const PYTHON_NODE_PATH_ROOT: &str = "python"; +pub(crate) const CURRENT_DIRECTORY: &str = "."; +pub(crate) const ROOT_DIRECTORY: &str = "/"; +pub(crate) const PLANNER_DEFAULT_CODE: &str = "S000"; +pub(crate) const UNKNOWN_NAME_CODE: &str = "S007"; +pub(crate) const UNKNOWN_TAG_CODE: &str = "S008"; +pub(crate) const UNKNOWN_PATH_CODE: &str = "S009"; +pub(crate) const UNMAPPED_KIND_CODE: &str = "S010"; +pub(crate) const PATH_BETWEEN_CODE: &str = "S011"; +pub(crate) const PATH_ROOT_CODE: &str = "S012"; +pub(crate) const UNIT_TEST_CODE: &str = "S013"; +pub(crate) const PATH_ROOT_ERROR: &str = + "path selectors require an explicit root: use 'models/' or 'python/'"; +pub(crate) const UNIT_TEST_ONLY_TEST_AND_BUILD: &str = + "only `sqb test` and `sqb build` accept unit-test selectors"; +/// Codes `sqb check` ignores for an exclude token that matches no Python node. +pub(crate) const IGNORED_EXCLUDE_CODES: [&str; 3] = + [UNKNOWN_NAME_CODE, UNKNOWN_TAG_CODE, UNKNOWN_PATH_CODE]; diff --git a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs new file mode 100644 index 0000000000..884bfa0b85 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs @@ -0,0 +1,4 @@ +pub mod python_node_graph_indexes; +pub mod resolve_python_node_excludes; +pub mod resolve_python_node_selectors; +pub mod resolve_python_sql_run_selection; diff --git a/crates/sqlbuild-analysis/src/python_selection/main/python_node_graph_indexes.rs b/crates/sqlbuild-analysis/src/python_selection/main/python_node_graph_indexes.rs new file mode 100644 index 0000000000..3aa4c74993 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/python_node_graph_indexes.rs @@ -0,0 +1,6 @@ +use crate::python_selection::models::{InventoryIndexes, InventoryNode}; + +/// Python's `PythonNodeGraph` edges, adjacency, tag and path indexes for discovered nodes. +pub fn python_node_graph_indexes(nodes: &[InventoryNode]) -> InventoryIndexes { + crate::python_selection::_helpers::inventory::indexes(nodes) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_node_excludes.rs b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_node_excludes.rs new file mode 100644 index 0000000000..02d13d8d01 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_node_excludes.rs @@ -0,0 +1,23 @@ +use crate::graph::errors::SelectorError; +use crate::python_selection::_helpers::nodes::select_nodes; +use crate::python_selection::constants::IGNORED_EXCLUDE_CODES; +use crate::python_selection::models::{NodeIndex, PythonSelectionGraph}; + +/// The Python node names exclude tokens name, skipping tokens that match nothing; sorted. +pub fn resolve_python_node_excludes( + graph: &PythonSelectionGraph, + exclude: &[String], +) -> Result, SelectorError> { + let index: NodeIndex<'_> = NodeIndex::new(graph); + let mut excluded: Vec = Vec::new(); + for token in exclude.iter().flat_map(|group| group.split_whitespace()) { + match select_nodes(&index, &[token.to_owned()], &[]) { + Ok(names) => excluded.extend(names), + Err(error) if IGNORED_EXCLUDE_CODES.contains(&error.code) => {} + Err(error) => return Err(error), + } + } + excluded.sort(); + excluded.dedup(); + Ok(excluded) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_node_selectors.rs b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_node_selectors.rs new file mode 100644 index 0000000000..bdc50693f2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_node_selectors.rs @@ -0,0 +1,14 @@ +use crate::graph::errors::SelectorError; +use crate::python_selection::_helpers::nodes::select_nodes; +use crate::python_selection::models::{NodeIndex, PythonSelectionGraph}; + +/// The Python node names `select` minus `exclude` resolve to, sorted; every node when unselected. +pub fn resolve_python_node_selectors( + graph: &PythonSelectionGraph, + select: &[String], + exclude: &[String], +) -> Result, SelectorError> { + Ok(select_nodes(&NodeIndex::new(graph), select, exclude)? + .into_iter() + .collect()) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs new file mode 100644 index 0000000000..a820fd41d7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs @@ -0,0 +1,21 @@ +use crate::graph::errors::SelectorError; +use crate::python_selection::_helpers::nodes::sql_names; +use crate::python_selection::_helpers::unified::run_selection; +use crate::python_selection::models::{ + NodeIndex, PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, +}; + +/// Python's `resolve_python_sql_run_selectors`: run selectors over SQL resources and Python nodes. +pub fn resolve_python_sql_run_selection( + sql: SqlSelectionGraph<'_>, + python: &PythonSelectionGraph, + select: &[String], + exclude: &[String], +) -> Result { + let context: SelectionContext<'_> = SelectionContext { + sql, + nodes: NodeIndex::new(python), + sql_names: sql_names(sql.graph), + }; + run_selection(&context, select, exclude) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/mod.rs b/crates/sqlbuild-analysis/src/python_selection/mod.rs new file mode 100644 index 0000000000..8e75a292cc --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/mod.rs @@ -0,0 +1,9 @@ +//! Selector resolution over Python nodes and SQL resources, and the Python node graph indexes. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/python_selection/models.rs b/crates/sqlbuild-analysis/src/python_selection/models.rs new file mode 100644 index 0000000000..b0d615e1dd --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/models.rs @@ -0,0 +1,89 @@ +//! Python node graphs, the SQL side they select against, and selection results. + +use std::collections::HashMap; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::models::ProjectGraph; + +/// One executable Python node as selectors see it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PythonSelectionNode { + pub name: String, + /// `task`, `asset`, `loader` or `check`. + pub kind: String, + /// The SQL resource keys the node declares, in authored order. + pub sql_deps: Vec, +} + +/// Python's `PythonNodeGraph` indexes, each in Python's dict order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PythonSelectionGraph { + pub nodes: Vec, + pub upstream: Vec<(String, Vec)>, + pub downstream: Vec<(String, Vec)>, + pub tags: Vec<(String, Vec)>, + /// Node name to the folder of its file relative to the project. + pub paths: Vec<(String, String)>, +} + +/// The SQL graph Python nodes are selected with, and each source's loader in project order. +#[derive(Clone, Copy, Debug)] +pub struct SqlSelectionGraph<'graph> { + pub graph: &'graph ProjectGraph, + /// `(source name, loader)` for every compiled source. + pub sources: &'graph [(String, Option)], +} + +/// The SQL keys and runnable Python node names a run selects, both sorted. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RunSelection { + pub sql_keys: Vec, + pub python_node_names: Vec, +} + +/// One discovered Python node as the graph inventory indexes it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InventoryNode { + pub name: String, + pub tags: Vec, + /// The node's file relative to the project, in POSIX form. + pub relative_path: String, + /// The Python nodes it depends on, in authored order. + pub upstream: Vec, +} + +/// Python's `PythonNodeGraph` edges and indexes, each in Python's dict order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct InventoryIndexes { + /// `(upstream, downstream)` dependency edges. + pub edges: Vec<(String, String)>, + pub upstream: Vec<(String, Vec)>, + pub downstream: Vec<(String, Vec)>, + pub tags: Vec<(String, Vec)>, + pub paths: Vec<(String, String)>, +} + +/// One selected resource: a SQL key or a Python node name. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) enum Atom { + Sql(ObjectKey), + Python(String), +} + +/// Python's `nodes_by_name` and edge and tag dicts over a Python node graph, last entry winning. +#[derive(Clone, Debug)] +pub(crate) struct NodeIndex<'graph> { + pub(crate) graph: &'graph PythonSelectionGraph, + pub(crate) by_name: HashMap<&'graph str, &'graph PythonSelectionNode>, + pub(crate) upstream: HashMap<&'graph str, &'graph [String]>, + pub(crate) downstream: HashMap<&'graph str, &'graph [String]>, + pub(crate) tags: HashMap<&'graph str, &'graph [String]>, +} + +/// Both graphs one unified selection reads, with the SQL selector names indexed. +#[derive(Clone, Debug)] +pub(crate) struct SelectionContext<'graph> { + pub(crate) sql: SqlSelectionGraph<'graph>, + pub(crate) nodes: NodeIndex<'graph>, + pub(crate) sql_names: HashMap<&'graph str, &'graph ObjectKey>, +} diff --git a/crates/sqlbuild-analysis/src/python_selection/tests.rs b/crates/sqlbuild-analysis/src/python_selection/tests.rs new file mode 100644 index 0000000000..95e26d2c9f --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_python_selection.rs"] +mod python_selection; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs new file mode 100644 index 0000000000..92b68329c3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs @@ -0,0 +1,198 @@ +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::main::build_project_graph::build_project_graph; +use crate::graph::models::{GraphResource, ProjectGraph}; +use crate::python_selection::main::python_node_graph_indexes::python_node_graph_indexes; +use crate::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; +use crate::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; +use crate::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection; +use crate::python_selection::models::{ + InventoryIndexes, InventoryNode, PythonSelectionGraph, PythonSelectionNode, RunSelection, + SqlSelectionGraph, +}; +use crate::python_selection::tests::test_types::{ExpectedRun, SelectorFailure}; + +type OwnedFailure = (String, String); +type OwnedRun = (Vec, Vec); + +/// `(name, kind, relative path, tags, upstream nodes, sql deps)`. +type NodeSpec = ( + &'static str, + &'static str, + &'static str, + &'static [&'static str], + &'static [&'static str], + &'static [(&'static str, &'static str)], +); + +const ORDERS_NODES: &[NodeSpec] = &[ + ( + "load_events", + "loader", + "python/loaders/events.py", + &[], + &["prepare_orders"], + &[], + ), + ( + "prepare_orders", + "task", + "python/tasks/orders.py", + &["orders", "daily"], + &[], + &[], + ), + ( + "export_orders", + "asset", + "python/assets/orders.py", + &["exports", "daily"], + &["prepare_orders"], + &[("model", "orders")], + ), + ( + "check_orders_export", + "check", + "python/checks/orders.py", + &["exports"], + &["export_orders"], + &[], + ), + ( + "raw_orders", + "loader", + "python/loaders/orders.py", + &[], + &[], + &[], + ), +]; + +fn owned(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +fn key(pair: &(&str, &str)) -> ObjectKey { + (pair.0.to_owned(), pair.1.to_owned()) +} + +fn failure(error: SelectorError) -> OwnedFailure { + (error.code.to_owned(), error.message) +} + +/// The inventory rows of the orders Python nodes. +fn inventory_nodes() -> Vec { + ORDERS_NODES + .iter() + .map(|(name, _, path, tags, upstream, _)| InventoryNode { + name: (*name).to_owned(), + tags: owned(tags), + relative_path: (*path).to_owned(), + upstream: owned(upstream), + }) + .collect() +} + +/// The orders Python node graph, indexed by the native inventory. +pub(super) fn orders_python_graph() -> PythonSelectionGraph { + let indexes: InventoryIndexes = python_node_graph_indexes(&inventory_nodes()); + PythonSelectionGraph { + nodes: ORDERS_NODES + .iter() + .map(|(name, kind, _, _, _, sql_deps)| PythonSelectionNode { + name: (*name).to_owned(), + kind: (*kind).to_owned(), + sql_deps: sql_deps.iter().map(key).collect(), + }) + .collect(), + upstream: indexes.upstream, + downstream: indexes.downstream, + tags: indexes.tags, + paths: indexes.paths, + } +} + +fn orders_sql_graph() -> ProjectGraph { + let raw_orders: ObjectKey = key(&("source", "raw_orders")); + build_project_graph(&[ + GraphResource { + key: key(&("model", "orders")), + deps: vec![raw_orders.clone()], + tags: owned(&["daily"]), + folder: Some(String::new()), + }, + GraphResource { + key: raw_orders, + deps: Vec::new(), + tags: Vec::new(), + folder: None, + }, + ]) +} + +/// The Python node names the selectors resolve to over the orders graph. +pub(super) fn node_names(select: &[&str], exclude: &[&str]) -> Result, OwnedFailure> { + resolve_python_node_selectors(&orders_python_graph(), &owned(select), &owned(exclude)) + .map_err(failure) +} + +/// The run selection over the orders SQL and Python graphs. +pub(super) fn run(select: &[&str], exclude: &[&str]) -> Result { + let graph: ProjectGraph = orders_sql_graph(); + let sources: Vec<(String, Option)> = + vec![("raw_orders".to_owned(), Some("raw_orders".to_owned()))]; + resolve_python_sql_run_selection( + SqlSelectionGraph { + graph: &graph, + sources: &sources, + }, + &orders_python_graph(), + &owned(select), + &owned(exclude), + ) + .map( + |RunSelection { + sql_keys, + python_node_names, + }| (sql_keys, python_node_names), + ) + .map_err(failure) +} + +/// An owned copy of an expected node-name result. +pub(super) fn expected_names( + expected: Result<&[&str], SelectorFailure>, +) -> Result, OwnedFailure> { + expected + .map(owned) + .map_err(|(code, message)| (code.to_owned(), message.to_owned())) +} + +/// An owned copy of an expected run result. +pub(super) fn expected_run( + expected: Result, +) -> Result { + expected + .map(|(keys, names)| (keys.iter().map(key).collect(), owned(names))) + .map_err(|(code, message)| (code.to_owned(), message.to_owned())) +} + +/// The folder the inventory indexes for one relative path. +pub(super) fn indexed_folder(relative_path: &str) -> String { + let nodes: Vec = vec![InventoryNode { + name: "node".to_owned(), + tags: Vec::new(), + relative_path: relative_path.to_owned(), + upstream: Vec::new(), + }]; + python_node_graph_indexes(&nodes) + .paths + .into_iter() + .map(|(_, folder)| folder) + .collect() +} + +/// The Python node names `sqb check` exclude tokens name over the orders graph. +pub(super) fn excluded_names(exclude: &[&str]) -> Result, OwnedFailure> { + resolve_python_node_excludes(&orders_python_graph(), &owned(exclude)).map_err(failure) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs new file mode 100644 index 0000000000..aabf3dc22f --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs @@ -0,0 +1,255 @@ +use crate::python_selection::tests::helpers::{ + excluded_names, expected_names, expected_run, indexed_folder, node_names, run, +}; +use crate::python_selection::tests::test_types::{ + InventoryTestCase, NodeSelectorTestCase, RunSelectionTestCase, +}; + +#[test] +fn given_python_node_selectors_when_resolving_then_names_match_python() { + let test_cases = [ + NodeSelectorTestCase { + description: "tag", + select: &["tag:daily"], + exclude: &[], + expected_names: Ok(&["export_orders", "prepare_orders"]), + }, + NodeSelectorTestCase { + description: "typed selector with upstream", + select: &["+asset:export_orders"], + exclude: &[], + expected_names: Ok(&["export_orders", "prepare_orders"]), + }, + NodeSelectorTestCase { + description: "explicit Python path", + select: &["path:python/tasks"], + exclude: &[], + expected_names: Ok(&["prepare_orders"]), + }, + NodeSelectorTestCase { + description: "downstream minus checks", + select: &["prepare_orders+"], + exclude: &["check:*"], + expected_names: Ok(&["export_orders", "load_events", "prepare_orders"]), + }, + NodeSelectorTestCase { + description: "intersection", + select: &["task:prepare_orders,tag:orders"], + exclude: &[], + expected_names: Ok(&["prepare_orders"]), + }, + NodeSelectorTestCase { + description: "path-between selector", + select: &["a~b"], + exclude: &[], + expected_names: Err(( + "S011", + "path-between selector 'a~b' is not supported for Python nodes yet", + )), + }, + NodeSelectorTestCase { + description: "path without a root", + select: &["path:other"], + exclude: &[], + expected_names: Err(( + "S012", + "path selectors require an explicit root: use 'models/' or 'python/'", + )), + }, + NodeSelectorTestCase { + description: "unknown tag", + select: &["tag:missing"], + exclude: &[], + expected_names: Err(("S008", "no Python nodes found with tag 'missing'")), + }, + NodeSelectorTestCase { + description: "SQL selector kind", + select: &["seed:countries"], + exclude: &[], + expected_names: Err(( + "S010", + "selector type 'seed' does not map to a Python node type", + )), + }, + NodeSelectorTestCase { + description: "unknown name", + select: &["missing"], + exclude: &[], + expected_names: Err(("S007", "unknown Python node selector 'missing'")), + }, + NodeSelectorTestCase { + description: "unit test", + select: &["test:orders"], + exclude: &[], + expected_names: Err(( + "S013", + "selector 'test:orders' selects a unit test; \ + only `sqb test` and `sqb build` accept unit-test selectors", + )), + }, + ]; + + for test_case in test_cases { + let names = node_names(test_case.select, test_case.exclude); + + assert_eq!( + names, + expected_names(test_case.expected_names), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_run_selectors_when_resolving_then_selection_matches_python() { + let test_cases = [ + RunSelectionTestCase { + description: "everything without checks, intermediate loader as a source", + select: &[], + exclude: &[], + expected_selection: Ok(( + &[ + ("model", "orders"), + ("source", "load_events"), + ("source", "raw_orders"), + ], + &[ + "export_orders", + "load_events", + "prepare_orders", + "raw_orders", + ], + )), + }, + RunSelectionTestCase { + description: "upstream reaches declared SQL and its terminal loader", + select: &["+export_orders"], + exclude: &[], + expected_selection: Ok(( + &[("model", "orders"), ("source", "raw_orders")], + &["export_orders", "prepare_orders", "raw_orders"], + )), + }, + RunSelectionTestCase { + description: "downstream reaches Python nodes declaring the SQL", + select: &["orders+ prepare_orders"], + exclude: &[], + expected_selection: Ok((&[("model", "orders")], &["export_orders", "prepare_orders"])), + }, + RunSelectionTestCase { + description: "checks are rejected", + select: &["check:check_orders_export"], + exclude: &[], + expected_selection: Err(( + "S000", + "Python checks are not selectable here: check_orders_export. \ + Use sqb check instead.", + )), + }, + RunSelectionTestCase { + description: "unselected upstream node", + select: &["load_events"], + exclude: &[], + expected_selection: Err(( + "S000", + "Python node 'load_events' depends on unselected Python node 'prepare_orders'; \ + select it directly or use upstream expansion", + )), + }, + RunSelectionTestCase { + description: "managed source loader selected as a loader", + select: &["loader:raw_orders"], + exclude: &[], + expected_selection: Err(( + "S007", + "loader selector 'raw_orders' matches managed source loader 'raw_orders'; \ + select it as source:raw_orders", + )), + }, + RunSelectionTestCase { + description: "tag on neither side", + select: &["tag:missing"], + exclude: &[], + expected_selection: Err(( + "S000", + "no SQL resources or Python nodes found with tag 'missing'", + )), + }, + ]; + + for test_case in test_cases { + let selection = run(test_case.select, test_case.exclude); + + assert_eq!( + selection, + expected_run(test_case.expected_selection), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_relative_paths_when_indexing_then_folders_match_python_parent() { + let test_cases = [ + InventoryTestCase { + description: "nested file", + relative_path: "python/tasks/orders.py", + expected_folder: "python/tasks", + }, + InventoryTestCase { + description: "top-level file", + relative_path: "orders.py", + expected_folder: ".", + }, + InventoryTestCase { + description: "file at the filesystem root", + relative_path: "/orders.py", + expected_folder: "/", + }, + ]; + + for test_case in test_cases { + let folder = indexed_folder(test_case.relative_path); + + assert_eq!( + folder, test_case.expected_folder, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_check_exclude_tokens_when_resolving_then_unmatched_tokens_are_skipped() { + let test_cases = [ + NodeSelectorTestCase { + description: "unknown name, tag and path are skipped", + select: &[], + exclude: &["missing tag:missing path:python/none", "tag:exports"], + expected_names: Ok(&["check_orders_export", "export_orders"]), + }, + NodeSelectorTestCase { + description: "other selector failures are raised", + select: &[], + exclude: &["path:other"], + expected_names: Err(( + "S012", + "path selectors require an explicit root: use 'models/' or 'python/'", + )), + }, + ]; + + for test_case in test_cases { + let names = excluded_names(test_case.exclude); + + assert_eq!( + names, + expected_names(test_case.expected_names), + "{} {:?}", + test_case.description, + test_case.select + ); + } +} diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_types.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_types.rs new file mode 100644 index 0000000000..67674006a1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_types.rs @@ -0,0 +1,28 @@ +pub(super) struct NodeSelectorTestCase { + pub(super) description: &'static str, + pub(super) select: &'static [&'static str], + pub(super) exclude: &'static [&'static str], + pub(super) expected_names: Result<&'static [&'static str], SelectorFailure>, +} + +pub(super) struct RunSelectionTestCase { + pub(super) description: &'static str, + pub(super) select: &'static [&'static str], + pub(super) exclude: &'static [&'static str], + pub(super) expected_selection: Result, +} + +pub(super) struct InventoryTestCase { + pub(super) description: &'static str, + pub(super) relative_path: &'static str, + pub(super) expected_folder: &'static str, +} + +/// `(sql keys, python node names)` a run selects. +pub(super) type ExpectedRun = ( + &'static [(&'static str, &'static str)], + &'static [&'static str], +); + +/// `(code, message)` of the `PlannerInputError` Python raises. +pub(super) type SelectorFailure = (&'static str, &'static str); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index bd9d3ff0ed..6d6a88e47f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -16,6 +16,7 @@ use crate::bindings::_helpers::check::check_results::{ check_result_text, results_json, write_results, }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::graph::python_selection::python_node_names; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; @@ -32,7 +33,6 @@ const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const PYTHON_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; -const PYTHON_SELECTORS_MODULE: &str = "sqlbuild.compiler.python_nodes.main.selectors"; const PYTHON_LIFECYCLE_MODULE: &str = "sqlbuild.compiler.python_nodes.main._run_lifecycle"; const PYTHON_NODE_MODELS: &str = "sqlbuild.compiler.python_nodes.models"; const PYTHON_NODE_TYPES: &str = "sqlbuild.compiler.python_nodes.types"; @@ -106,18 +106,7 @@ fn python_nodes<'py>( graph: &Bound<'py, PyAny>, (select, exclude): (Bound<'py, PyAny>, Bound<'py, PyAny>), ) -> PyResult> { - call_with( - &imported( - graph.py(), - PYTHON_SELECTORS_MODULE, - "resolve_python_nodes_from_selectors", - )?, - &[ - ("select", select), - ("exclude", exclude), - ("graph", graph.clone()), - ], - ) + Ok(python_node_names(graph, (select.extract()?, exclude.extract()?))?.into_any()) } fn node_kind<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs index 8b8fc4bab3..5e3f7e22ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs @@ -2,5 +2,6 @@ mod algorithms; mod execution_gates; pub(crate) mod keys; pub(crate) mod project_graph; +pub(crate) mod python_selection; pub(crate) mod registration; pub(crate) mod views; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs new file mode 100644 index 0000000000..b61ae69be3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -0,0 +1,217 @@ +//! Python node and unified SQL/Python selector resolution, and the Python node graph indexes. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyFrozenSet}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::errors::SelectorError; +use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; +use sqlbuild_analysis::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; +use sqlbuild_analysis::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; +use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection as run_selection; +use sqlbuild_analysis::python_selection::models::{ + InventoryIndexes, InventoryNode, PythonSelectionGraph, PythonSelectionNode, RunSelection, + SqlSelectionGraph, +}; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; + +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; + +type NameRows = Vec<(String, Vec)>; +/// `(name, tags, POSIX relative path, upstream node names)` of one discovered node. +type InventoryRow = (String, Vec, String, Vec); +/// `(edges, upstream, downstream, tags, paths)` in Python's dict order. +type IndexesRow = ( + Vec<(String, String)>, + NameRows, + NameRows, + NameRows, + Vec<(String, String)>, +); + +/// The `PlannerInputError` Python raises for a selector failure. +fn planner_error(py: Python<'_>, error: SelectorError) -> PyErr { + let raised = (|| -> PyResult> { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("code", error.code)?; + keywords.set_item("help", error.help)?; + imported(py, PLANNER_EXCEPTIONS_MODULE, "PlannerInputError")? + .call((error.message,), Some(&keywords)) + })(); + match raised { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +fn strings(values: &Bound<'_, PyAny>) -> PyResult> { + values.try_iter()?.map(|value| value?.extract()).collect() +} + +/// A `dict[str, Iterable[str]]` as rows in dict order. +fn name_rows(mapping: &Bound<'_, PyAny>) -> PyResult { + let mut rows: NameRows = Vec::new(); + for item in mapping.call_method0("items")?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + rows.push((item.get_item(0)?.extract()?, strings(&item.get_item(1)?)?)); + } + Ok(rows) +} + +fn selection_node(node: &Bound<'_, PyAny>) -> PyResult { + let mut sql_deps: Vec = Vec::new(); + for reference in node.getattr("sql_deps")?.try_iter()? { + let reference: Bound<'_, PyAny> = reference?; + sql_deps.push(( + reference.getattr("kind")?.getattr("value")?.extract()?, + reference.getattr("name")?.extract()?, + )); + } + Ok(PythonSelectionNode { + name: node.getattr("name")?.extract()?, + kind: node.getattr("kind")?.getattr("value")?.extract()?, + sql_deps, + }) +} + +/// A `PythonNodeGraph` as the native selection graph. +fn selection_graph(graph: &Bound<'_, PyAny>) -> PyResult { + Ok(PythonSelectionGraph { + nodes: graph + .getattr("nodes")? + .try_iter()? + .map(|node| selection_node(&node?)) + .collect::>()?, + upstream: name_rows(&graph.getattr("upstream_deps")?)?, + downstream: name_rows(&graph.getattr("downstream_deps")?)?, + tags: name_rows(&graph.getattr("tag_index")?)?, + paths: graph + .getattr("path_index")? + .call_method0("items")? + .try_iter()? + .map(|item| item?.extract()) + .collect::>()?, + }) +} + +/// `(name, loader)` of every source in the compiled project. +fn project_sources(project_graph: &Bound<'_, PyAny>) -> PyResult)>> { + project_graph + .getattr("project")? + .getattr("sources")? + .try_iter()? + .map(|source| { + let source: Bound<'_, PyAny> = source?; + Ok(( + source.getattr("name")?.extract()?, + source + .getattr("source_entry")? + .getattr("loader")? + .extract()?, + )) + }) + .collect() +} + +/// The Python node names `select` minus `exclude` resolve to; every node when unselected. +pub(crate) fn python_node_names<'py>( + graph: &Bound<'py, PyAny>, + (select, exclude): (Vec, Vec), +) -> PyResult> { + let py: Python<'py> = graph.py(); + let native: PythonSelectionGraph = selection_graph(graph)?; + let names: Vec = + compiler_guard(|| Ok(resolve_python_node_selectors(&native, &select, &exclude)))? + .map_err(|error| planner_error(py, error))?; + PyFrozenSet::new(py, names) +} + +/// The Python node names `select` minus `exclude` resolve to; every node when unselected. +#[pyfunction] +#[pyo3(signature = (*, graph, select, exclude))] +fn resolve_python_node_selection<'py>( + graph: &Bound<'py, PyAny>, + select: Vec, + exclude: Vec, +) -> PyResult> { + python_node_names(graph, (select, exclude)) +} + +/// The Python node names `sqb check` exclude tokens name, skipping tokens that match nothing. +#[pyfunction] +#[pyo3(signature = (*, graph, exclude))] +fn resolve_python_check_excludes<'py>( + graph: &Bound<'py, PyAny>, + exclude: Vec, +) -> PyResult> { + let py: Python<'py> = graph.py(); + let native: PythonSelectionGraph = selection_graph(graph)?; + let names: Vec = + compiler_guard(|| Ok(resolve_python_node_excludes(&native, &exclude)))? + .map_err(|error| planner_error(py, error))?; + PyFrozenSet::new(py, names) +} + +/// `(sql keys, python node names)` run selectors resolve to across SQL resources and Python nodes. +#[pyfunction] +#[pyo3(signature = (*, project_graph, python_graph, select, exclude))] +fn resolve_python_sql_run_selection( + project_graph: &Bound<'_, PyAny>, + python_graph: &Bound<'_, PyAny>, + select: Vec, + exclude: Vec, +) -> PyResult<(Vec, Vec)> { + let py: Python<'_> = project_graph.py(); + let native: Bound<'_, NativeProjectGraph> = project_graph.getattr("native")?.downcast_into()?; + let sources: Vec<(String, Option)> = project_sources(project_graph)?; + let nodes: PythonSelectionGraph = selection_graph(python_graph)?; + let selection: RunSelection = compiler_guard(|| { + Ok(run_selection( + SqlSelectionGraph { + graph: native.get().graph(), + sources: &sources, + }, + &nodes, + &select, + &exclude, + )) + })? + .map_err(|error| planner_error(py, error))?; + Ok((selection.sql_keys, selection.python_node_names)) +} + +/// The dependency edges, adjacency, tag and path indexes of discovered Python nodes. +#[pyfunction] +fn python_node_graph_indexes(nodes: Vec) -> PyResult { + compiler_guard(|| { + let nodes: Vec = nodes + .into_iter() + .map(|(name, tags, relative_path, upstream)| InventoryNode { + name, + tags, + relative_path, + upstream, + }) + .collect(); + let InventoryIndexes { + edges, + upstream, + downstream, + tags, + paths, + } = graph_indexes(&nodes); + Ok((edges, upstream, downstream, tags, paths)) + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_python_node_selection, module)?)?; + module.add_function(wrap_pyfunction!(resolve_python_check_excludes, module)?)?; + module.add_function(wrap_pyfunction!(resolve_python_sql_run_selection, module)?)?; + module.add_function(wrap_pyfunction!(python_node_graph_indexes, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs index 2140dbce21..485560f171 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs @@ -4,5 +4,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::graph::project_graph::register(module)?; crate::bindings::_helpers::graph::algorithms::register(module)?; crate::bindings::_helpers::graph::execution_gates::register(module)?; + crate::bindings::_helpers::graph::python_selection::register(module)?; crate::bindings::_helpers::graph::views::register(module) } diff --git a/python-boundary.toml b/python-boundary.toml index 5428df2b72..375fee0492 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 66074 +remaining_lines = 64690 [[stays]] path = "executor" @@ -189,6 +189,11 @@ path = "compiler/python_nodes/_helpers/identity.py" reason = "user-code" note = "Identity hashing over live user functions (source and globals)." +[[stays]] +path = "compiler/python_nodes/_helpers/user_inventory.py" +reason = "user-code" +note = "Builds Python nodes from loaded user functions: signatures, provider annotations and decorated dependencies." + [[stays]] path = "compiler/discovery/_helpers/filesystem/core.py" reason = "user-code" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4048a0c04e..5090770383 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3182,6 +3182,28 @@ def attached_audit_gate_edges( ) -> tuple[AttachedAuditGateEdge, ...]: ... def hook_read_edges(*, project: CompiledProjectView) -> tuple[HookReadEdge, ...]: ... def model_python_hook_names(*, model: CompiledModelView) -> tuple[str, ...]: ... +def resolve_python_node_selection( + *, graph: PythonNodeGraph, select: Sequence[str], exclude: Sequence[str] +) -> frozenset[str]: ... +def resolve_python_check_excludes( + *, graph: PythonNodeGraph, exclude: Sequence[str] +) -> frozenset[str]: ... +def resolve_python_sql_run_selection( + *, + project_graph: ProjectGraph, + python_graph: PythonNodeGraph, + select: Sequence[str], + exclude: Sequence[str], +) -> tuple[list[tuple[str, str]], list[str]]: ... +def python_node_graph_indexes( + nodes: list[tuple[str, tuple[str, ...], str, list[str]]], / +) -> tuple[ + list[tuple[str, str]], + list[tuple[str, list[str]]], + list[tuple[str, list[str]]], + list[tuple[str, list[str]]], + list[tuple[str, str]], +]: ... def sql_ref_key(reference: SqlResourceRef, /) -> CompiledObjectKey: ... def native_graph_key(key: CompiledObjectKey, /) -> tuple[str, str]: ... def native_graph_keys(keys: Iterable[CompiledObjectKey], /) -> list[tuple[str, str]]: ... diff --git a/src/sqlbuild/cli/commands/_helpers/check/core.py b/src/sqlbuild/cli/commands/_helpers/check/core.py index 3ff34e394d..994fcfb4f9 100644 --- a/src/sqlbuild/cli/commands/_helpers/check/core.py +++ b/src/sqlbuild/cli/commands/_helpers/check/core.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( build_python_relation_targets, + resolve_python_check_excludes, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationLookup @@ -18,8 +19,6 @@ from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction, DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.python_nodes.main.selectors import resolve_python_nodes_from_selectors from sqlbuild.compiler.python_nodes.models import ( PythonNodeGraph, PythonSqlRunLifecyclePlan, @@ -39,8 +38,6 @@ from sqlbuild.python_nodes.types import SqlResourceRefKind from sqlbuild.spec.contracts.models import SourceEntry -_IGNORED_EXCLUDE_SELECTOR_ERROR_CODES: frozenset[str] = frozenset({"S007", "S008", "S009"}) - def record_python_run_state_results( *, @@ -203,8 +200,8 @@ def relevant_check_functions( ) -> tuple[DiscoveredCheckFunction, ...]: """Return non-excluded checks whose Python and SQL dependencies all ran in this build.""" - excluded_names: frozenset[str] = _resolve_python_check_excludes( - exclude=exclude, python_graph=python_graph + excluded_names: frozenset[str] = resolve_python_check_excludes( + graph=python_graph, exclude=exclude ) check_names: set[str] = set() for node in python_graph.nodes: @@ -223,27 +220,6 @@ def relevant_check_functions( return tuple(check for check in discovered_inputs.check_functions if check.name in check_names) -def _resolve_python_check_excludes( - *, exclude: tuple[str, ...], python_graph: PythonNodeGraph -) -> frozenset[str]: - excluded: set[str] = set() - raw_exclude: str - for raw_exclude in exclude: - token: str - for token in raw_exclude.split(): - try: - excluded.update( - resolve_python_nodes_from_selectors( - select=(token,), exclude=(), graph=python_graph - ) - ) - except PlannerInputError as error: - if error.code in _IGNORED_EXCLUDE_SELECTOR_ERROR_CODES: - continue - raise - return frozenset(excluded) - - def _load_result_to_python_result( *, node_name: str, result: LoadExecutionResult ) -> PythonNodeExecutionResult: diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/run_selectors.py b/src/sqlbuild/compiler/python_nodes/_helpers/run_selectors.py deleted file mode 100644 index ded78cd0b9..0000000000 --- a/src/sqlbuild/compiler/python_nodes/_helpers/run_selectors.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Run-command selector helpers for SQL resources and executable Python nodes.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.python_nodes._helpers.unified_selectors import ( - resolve_python_sql_selectors, - validate_python_sql_selection_dependencies, -) -from sqlbuild.compiler.python_nodes.models import ( - PythonNodeGraph, - PythonSqlRunSelection, - PythonSqlSelection, -) -from sqlbuild.compiler.python_nodes.types import PythonNodeKind - - -def resolve_python_sql_run_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - project_graph: ProjectGraph, - python_graph: PythonNodeGraph, -) -> PythonSqlRunSelection: - """Resolve build-style selectors across SQL resources and Python nodes.""" - - selection: PythonSqlSelection = resolve_python_sql_selectors( - select=select, - exclude=exclude, - project_graph=project_graph, - python_graph=python_graph, - validate_dependencies=False, - ) - selected_check_names: frozenset[str] = frozenset( - name - for name in selection.python_node_names - if python_graph.nodes_by_name[name].kind == PythonNodeKind.CHECK - ) - if select and selected_check_names: - check_list: str = ", ".join(sorted(selected_check_names)) - raise PlannerInputError( - f"Python checks are not selectable here: {check_list}. Use sqb check instead." - ) - runnable_python_names: frozenset[str] = frozenset( - name - for name in selection.python_node_names - if python_graph.nodes_by_name[name].kind != PythonNodeKind.CHECK - ) - run_selection: PythonSqlRunSelection = PythonSqlRunSelection( - sql_keys=selection.sql_keys, - python_node_names=runnable_python_names, - ) - validate_python_sql_selection_dependencies( - selection=PythonSqlSelection( - sql_keys=run_selection.sql_keys, - python_node_names=run_selection.python_node_names, - ), - project_graph=project_graph, - python_graph=python_graph, - ) - return PythonSqlRunSelection( - sql_keys=run_selection.sql_keys - | frozenset( - CompiledObjectKey(resource_type=CompiledResourceType.SOURCE, name=name) - for name in run_selection.python_node_names - if python_graph.nodes_by_name[name].kind == PythonNodeKind.LOADER - and name not in _terminal_loader_names(project_graph=project_graph) - ), - python_node_names=run_selection.python_node_names, - ) - - -def _terminal_loader_names(*, project_graph: ProjectGraph) -> frozenset[str]: - return frozenset( - source.source_entry.loader - for source in project_graph.project.sources - if source.source_entry.loader is not None - ) diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/selectors.py b/src/sqlbuild/compiler/python_nodes/_helpers/selectors.py deleted file mode 100644 index 276cf1fcf2..0000000000 --- a/src/sqlbuild/compiler/python_nodes/_helpers/selectors.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Selector helpers for executable Python DAG nodes.""" - -from __future__ import annotations - -from fnmatch import fnmatchcase - -from sqlbuild._native import transitive_closure_many -from sqlbuild.compiler.planner.constants import PATH_SELECTOR_EXPLICIT_ROOT_ERROR -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.selection.selector_parse import parse_project_selector -from sqlbuild.compiler.planner.main.selection.unit_test_selector_error import ( - unit_test_selector_error, -) -from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector -from sqlbuild.compiler.planner.types import SelectorKind -from sqlbuild.compiler.python_nodes.constants import ( - EMPTY_SELECTOR_FOLDER, - UNIFIED_PATH_ROOTS, -) -from sqlbuild.compiler.python_nodes.models import DiscoveredPythonNode, PythonNodeGraph -from sqlbuild.compiler.python_nodes.types import PythonNodeKind - -_PYTHON_NODE_KIND_BY_SELECTOR_KIND: dict[SelectorKind, PythonNodeKind] = { - SelectorKind.TASK: PythonNodeKind.TASK, - SelectorKind.ASSET: PythonNodeKind.ASSET, - SelectorKind.LOADER: PythonNodeKind.LOADER, - SelectorKind.CHECK: PythonNodeKind.CHECK, -} - - -def resolve_python_node_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - graph: PythonNodeGraph, -) -> frozenset[str]: - """Resolve raw selectors into Python-node names.""" - - if not select: - return frozenset(graph.nodes_by_name) - - selected: set[str] = set() - raw_select: str - for raw_select in select: - token: str - for token in raw_select.split(): - selected.update(_resolve_token(token=token, graph=graph)) - - excluded: set[str] = set() - raw_exclude: str - for raw_exclude in exclude: - token: str - for token in raw_exclude.split(): - excluded.update(_resolve_token(token=token, graph=graph)) - - return frozenset(selected - excluded) - - -def _resolve_token(*, token: str, graph: PythonNodeGraph) -> frozenset[str]: - parts: list[str] = token.split(",") - if len(parts) == 1: - return _resolve_single(raw=parts[0], graph=graph) - - sets: list[frozenset[str]] = [_resolve_single(raw=part, graph=graph) for part in parts] - result: frozenset[str] = sets[0] - subsequent: frozenset[str] - for subsequent in sets[1:]: - result = result & subsequent - return result - - -def _resolve_single(*, raw: str, graph: PythonNodeGraph) -> frozenset[str]: - parsed: ParsedSelector | PathSelector = parse_project_selector(raw) - if isinstance(parsed, PathSelector): - raise PlannerInputError( - f"path-between selector '{raw}' is not supported for Python nodes yet", - code="S011", - ) - - if parsed.kind == SelectorKind.TEST: - raise unit_test_selector_error(selector=raw) - if parsed.kind == SelectorKind.TAG: - return _resolve_tag(parsed=parsed, graph=graph) - - if parsed.kind == SelectorKind.PATH: - return _resolve_path(parsed=parsed, graph=graph) - - node_names: frozenset[str] = _lookup_node_names(parsed=parsed, graph=graph) - if not node_names: - raise PlannerInputError(f"unknown Python node selector '{parsed.value}'", code="S007") - - return _expand_selected(names=node_names, parsed=parsed, graph=graph) - - -def _resolve_path(*, parsed: ParsedSelector, graph: PythonNodeGraph) -> frozenset[str]: - folder: str = parsed.value.replace("\\", "/").strip("/") - _validate_python_path_root(folder) - matched_names: frozenset[str] = frozenset( - name - for name, node_folder in graph.path_index.items() - if _path_matches(indexed_folder=node_folder, selector_folder=folder) - ) - if not matched_names: - raise PlannerInputError(f"no Python nodes found under path '{folder}'", code="S009") - - return _expand_selected(names=matched_names, parsed=parsed, graph=graph) - - -def _path_matches(*, indexed_folder: str, selector_folder: str) -> bool: - if selector_folder == EMPTY_SELECTOR_FOLDER: - return True - return indexed_folder == selector_folder or indexed_folder.startswith(f"{selector_folder}/") - - -def _validate_python_path_root(folder: str) -> None: - root: str = folder.split("/", 1)[0] - if root in UNIFIED_PATH_ROOTS: - return - raise PlannerInputError( - PATH_SELECTOR_EXPLICIT_ROOT_ERROR, - code="S012", - ) - - -def _resolve_tag(*, parsed: ParsedSelector, graph: PythonNodeGraph) -> frozenset[str]: - tagged_names: frozenset[str] = graph.tag_index.get(parsed.value, frozenset()) - if not tagged_names: - raise PlannerInputError(f"no Python nodes found with tag '{parsed.value}'", code="S008") - - return _expand_selected(names=tagged_names, parsed=parsed, graph=graph) - - -def _lookup_node_names(*, parsed: ParsedSelector, graph: PythonNodeGraph) -> frozenset[str]: - if parsed.kind == SelectorKind.NAME: - if not _is_name_pattern(parsed.value): - node: DiscoveredPythonNode | None = graph.nodes_by_name.get(parsed.value) - return frozenset() if node is None else frozenset((node.name,)) - return frozenset( - node.name - for name, node in graph.nodes_by_name.items() - if fnmatchcase(name, parsed.value) - ) - - python_node_kind: PythonNodeKind | None = _PYTHON_NODE_KIND_BY_SELECTOR_KIND.get(parsed.kind) - if python_node_kind is None: - raise PlannerInputError( - f"selector type '{parsed.kind}' does not map to a Python node type", - code="S010", - ) - - if not _is_name_pattern(parsed.value): - node = graph.nodes_by_typed_selector.get(f"{python_node_kind.value}:{parsed.value}") - return frozenset() if node is None else frozenset((node.name,)) - return frozenset( - node.name - for node in graph.nodes_by_name.values() - if node.kind == python_node_kind and fnmatchcase(node.name, parsed.value) - ) - - -def _is_name_pattern(value: str) -> bool: - return any(character in value for character in "*?[") - - -def _expand_selected( - *, names: frozenset[str], parsed: ParsedSelector, graph: PythonNodeGraph -) -> frozenset[str]: - result: set[str] = set(names) - if parsed.upstream: - result.update( - transitive_closure_many(starts=names, edges=graph.upstream_deps, include_starts=False) - ) - if parsed.downstream: - result.update( - transitive_closure_many(starts=names, edges=graph.downstream_deps, include_starts=False) - ) - return frozenset(result) diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py b/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py deleted file mode 100644 index 0c5207de3e..0000000000 --- a/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py +++ /dev/null @@ -1,675 +0,0 @@ -"""Unified selector helpers for SQL resources and executable Python nodes.""" - -from __future__ import annotations - -from dataclasses import dataclass -from fnmatch import fnmatchcase - -from sqlbuild._native import sql_ref_key, transitive_closure, transitive_closure_many -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.constants import ( - PATH_SELECTOR_EXPLICIT_ROOT_ERROR, - SELECTOR_EXPANSION_MARKER, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.selection._build_resources import ( - expand_build_resource_selection, -) -from sqlbuild.compiler.planner.main.selection.selector_expansion import split_selector_expansion -from sqlbuild.compiler.planner.main.selection.selector_parse import parse_project_selector -from sqlbuild.compiler.planner.main.selection.unit_test_selector_error import ( - unit_test_selector_error, -) -from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector -from sqlbuild.compiler.planner.types import SelectorKind -from sqlbuild.compiler.python_nodes._helpers.selectors import resolve_python_node_selectors -from sqlbuild.compiler.python_nodes.constants import ( - PYTHON_NODE_PATH_ROOTS, - SQL_MODEL_PATH_ROOT, - TAG_NOT_FOUND_ERROR_CODE, -) -from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonNode, - PythonNodeGraph, - PythonSqlSelection, -) -from sqlbuild.compiler.python_nodes.types import PythonNodeKind - -_PYTHON_SELECTOR_KINDS: frozenset[SelectorKind] = frozenset( - { - SelectorKind.TASK, - SelectorKind.ASSET, - SelectorKind.LOADER, - SelectorKind.CHECK, - } -) -_SQL_SELECTOR_KINDS: frozenset[SelectorKind] = frozenset( - { - SelectorKind.SEED, - SelectorKind.SOURCE, - SelectorKind.PATH, - } -) -_ALLOWED_SQL_MODEL_DEP_RESOURCE_TYPES: frozenset[CompiledResourceType] = frozenset( - { - CompiledResourceType.MODEL, - CompiledResourceType.SOURCE, - CompiledResourceType.SEED, - CompiledResourceType.UDF, - CompiledResourceType.TABLE_FN, - CompiledResourceType.DBT_REF, - } -) - - -@dataclass(frozen=True) -class _SelectionAtom: - kind: str - value: CompiledObjectKey | str - - -def resolve_python_sql_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - project_graph: ProjectGraph, - python_graph: PythonNodeGraph, - validate_dependencies: bool = True, -) -> PythonSqlSelection: - """Resolve selectors across compiled SQL resources and executable Python nodes.""" - - validate_python_sql_boundaries(project_graph=project_graph, python_graph=python_graph) - - selected: set[_SelectionAtom] - if select: - selected = _resolve_selector_groups( - raw_groups=select, - project_graph=project_graph, - python_graph=python_graph, - ) - else: - selected = { - *(_sql_atom(key) for key in project_graph.all_keys.values()), - *(_python_atom(name) for name in python_graph.nodes_by_name), - } - - excluded: set[_SelectionAtom] = _resolve_selector_groups( - raw_groups=exclude, - project_graph=project_graph, - python_graph=python_graph, - ) - selected -= excluded - selected.update( - _required_sql_resource_atoms(selected_atoms=selected, project_graph=project_graph) - ) - if validate_dependencies: - _validate_selected_dependencies( - selected_atoms=selected, project_graph=project_graph, python_graph=python_graph - ) - return _build_selection(selected) - - -def validate_python_sql_selection_dependencies( - *, - selection: PythonSqlSelection, - project_graph: ProjectGraph, - python_graph: PythonNodeGraph, -) -> None: - """Validate that a resolved SQL/Python selection contains required Python deps.""" - - _validate_selected_dependencies( - selected_atoms={ - *(_sql_atom(key) for key in selection.sql_keys), - *(_python_atom(name) for name in selection.python_node_names), - }, - project_graph=project_graph, - python_graph=python_graph, - ) - - -def validate_python_sql_boundaries( - *, project_graph: ProjectGraph, python_graph: PythonNodeGraph -) -> None: - """Validate SQL/Python graph boundary rules before unified selection/execution.""" - - _validate_sql_model_dependencies(project_graph=project_graph) - _validate_python_sql_refs(project_graph=project_graph, python_graph=python_graph) - _validate_loader_upstream_python_is_pre_sql(python_graph=python_graph) - terminal_loader_by_name: dict[str, str] = _terminal_loader_by_name(project_graph=project_graph) - node: DiscoveredPythonNode - for node in python_graph.nodes: - dependency_name: str - for dependency_name in python_graph.upstream_deps.get(node.name, ()): - source_name: str | None = terminal_loader_by_name.get(dependency_name) - if source_name is None: - continue - if node.kind == PythonNodeKind.CHECK: - raise PlannerInputError( - f"Check '{node.name}' depends on terminal loader '{dependency_name}'; " - f"use source audits for source '{source_name}' instead" - ) - if node.kind in {PythonNodeKind.TASK, PythonNodeKind.ASSET}: - raise PlannerInputError( - f"Python node '{node.name}' depends on terminal loader '{dependency_name}'; " - f"depend on source '{source_name}' instead" - ) - - -def _validate_loader_upstream_python_is_pre_sql(*, python_graph: PythonNodeGraph) -> None: - node: DiscoveredPythonNode - for node in python_graph.nodes: - if node.kind != PythonNodeKind.LOADER: - continue - upstream_name: str - for upstream_name in transitive_closure(start=node.name, edges=python_graph.upstream_deps): - upstream_node: DiscoveredPythonNode = python_graph.nodes_by_name[upstream_name] - if upstream_node.kind not in {PythonNodeKind.TASK, PythonNodeKind.ASSET}: - continue - if upstream_node.sql_deps: - raise PlannerInputError( - f"Loader '{node.name}' depends on Python node '{upstream_node.name}' " - "which depends on SQL; Python-to-SQL writes must flow through a " - "pre-SQL task/asset -> loader -> source path" - ) - - -def _validate_python_sql_refs( - *, project_graph: ProjectGraph, python_graph: PythonNodeGraph -) -> None: - node: DiscoveredPythonNode - for node in python_graph.nodes: - for sql_ref in node.sql_deps: - sql_key: CompiledObjectKey | None = project_graph.all_keys.get(sql_ref.name) - if sql_key is None: - raise PlannerInputError( - f"Python node '{node.name}' depends on unknown SQL resource '{sql_ref.name}'" - ) - if sql_key.resource_type != sql_ref_key(sql_ref).resource_type: - raise PlannerInputError( - f"Python node '{node.name}' declares {sql_ref.kind.value}('{sql_ref.name}') " - f"but '{sql_ref.name}' is a {sql_key.resource_type}" - ) - - -def _validate_sql_model_dependencies(*, project_graph: ProjectGraph) -> None: - source_names: frozenset[str] = frozenset( - source.name for source in project_graph.project.sources - ) - model_key: CompiledObjectKey - for model_key, dependency_keys in project_graph.upstream_deps.items(): - if model_key.resource_type != CompiledResourceType.MODEL: - continue - dependency_key: CompiledObjectKey - for dependency_key in dependency_keys: - if dependency_key.resource_type not in _ALLOWED_SQL_MODEL_DEP_RESOURCE_TYPES: - raise PlannerInputError( - f"SQL model '{model_key.name}' depends on non-SQL resource " - f"'{dependency_key.resource_type}:{dependency_key.name}'; SQL model " - "dependencies must stay SQL-only" - ) - if ( - dependency_key.resource_type == CompiledResourceType.SOURCE - and dependency_key.name not in source_names - ): - raise PlannerInputError( - f"SQL model '{model_key.name}' depends on intermediate loader " - f"'{dependency_key.name}'; depend on a source populated by a terminal " - "loader instead" - ) - - -def _resolve_selector_groups( - *, - raw_groups: tuple[str, ...], - project_graph: ProjectGraph, - python_graph: PythonNodeGraph, -) -> set[_SelectionAtom]: - resolved: set[_SelectionAtom] = set() - raw_group: str - for raw_group in raw_groups: - token: str - for token in raw_group.split(): - resolved.update( - _resolve_token( - token=token, - project_graph=project_graph, - python_graph=python_graph, - ) - ) - return resolved - - -def _resolve_token( - *, token: str, project_graph: ProjectGraph, python_graph: PythonNodeGraph -) -> frozenset[_SelectionAtom]: - parts: list[str] = token.split(",") - if len(parts) == 1: - return _resolve_single(raw=parts[0], project_graph=project_graph, python_graph=python_graph) - - resolved_parts: list[frozenset[_SelectionAtom]] = [ - _resolve_single(raw=part, project_graph=project_graph, python_graph=python_graph) - for part in parts - ] - result: frozenset[_SelectionAtom] = resolved_parts[0] - subsequent: frozenset[_SelectionAtom] - for subsequent in resolved_parts[1:]: - result = result & subsequent - return result - - -def _resolve_single( - *, raw: str, project_graph: ProjectGraph, python_graph: PythonNodeGraph -) -> frozenset[_SelectionAtom]: - parsed: ParsedSelector | PathSelector = parse_project_selector(raw) - if isinstance(parsed, PathSelector) or not (parsed.upstream or parsed.downstream): - return _resolve_single_side(raw=raw, project_graph=project_graph, python_graph=python_graph) - core: str = split_selector_expansion(raw).core - atoms: frozenset[_SelectionAtom] = frozenset() - if parsed.upstream: - upstream: frozenset[_SelectionAtom] = _resolve_single_side( - raw=f"{SELECTOR_EXPANSION_MARKER}{core}", - project_graph=project_graph, - python_graph=python_graph, - ) - atoms = ( - atoms - | upstream - | _sql_dependency_upstream_atoms( - atoms=upstream, project_graph=project_graph, python_graph=python_graph - ) - ) - if parsed.downstream: - downstream: frozenset[_SelectionAtom] = _resolve_single_side( - raw=f"{core}{SELECTOR_EXPANSION_MARKER}", - project_graph=project_graph, - python_graph=python_graph, - ) - atoms = ( - atoms - | downstream - | _sql_dependent_python_atoms(atoms=downstream, python_graph=python_graph) - ) - return atoms - - -def _sql_dependency_upstream_atoms( - *, - atoms: frozenset[_SelectionAtom], - project_graph: ProjectGraph, - python_graph: PythonNodeGraph, -) -> frozenset[_SelectionAtom]: - """Return the SQL resources selected Python nodes declare, with their SQL upstream.""" - - dependency_keys: set[CompiledObjectKey] = set() - atom: _SelectionAtom - for atom in atoms: - if isinstance(atom.value, str): - dependency_keys.update(_sql_dependency_keys(python_graph.nodes_by_name[atom.value])) - if not dependency_keys: - return frozenset() - sql_atoms: set[_SelectionAtom] = { - _sql_atom(key) - for key in transitive_closure_many( - starts=dependency_keys, edges=project_graph.upstream_deps, include_starts=True - ) - } - return frozenset( - sql_atoms - | _required_terminal_loader_atoms( - selected_atoms=sql_atoms, project_graph=project_graph, python_graph=python_graph - ) - ) - - -def _sql_dependent_python_atoms( - *, atoms: frozenset[_SelectionAtom], python_graph: PythonNodeGraph -) -> frozenset[_SelectionAtom]: - """Return runnable Python nodes that declare a selected SQL resource, with their downstream.""" - - selected_keys: frozenset[CompiledObjectKey] = frozenset( - atom.value for atom in atoms if isinstance(atom.value, CompiledObjectKey) - ) - dependent_names: frozenset[str] = frozenset( - node.name - for node in python_graph.nodes - if node.kind != PythonNodeKind.CHECK - and not _sql_dependency_keys(node).isdisjoint(selected_keys) - ) - if not dependent_names: - return frozenset() - return frozenset( - _python_atom(name) - for name in transitive_closure_many( - starts=dependent_names, edges=python_graph.downstream_deps, include_starts=True - ) - if python_graph.nodes_by_name[name].kind != PythonNodeKind.CHECK - ) - - -def _sql_dependency_keys(node: DiscoveredPythonNode) -> frozenset[CompiledObjectKey]: - return frozenset(sql_ref_key(sql_ref) for sql_ref in node.sql_deps) - - -def _resolve_single_side( - *, raw: str, project_graph: ProjectGraph, python_graph: PythonNodeGraph -) -> frozenset[_SelectionAtom]: - parsed: ParsedSelector | PathSelector = parse_project_selector(raw) - atoms: frozenset[_SelectionAtom] - if isinstance(parsed, PathSelector): - atoms = _resolve_sql(raw=raw, project_graph=project_graph) - if parsed.upstream: - atoms = atoms | frozenset( - _required_terminal_loader_atoms( - selected_atoms=set(atoms), - project_graph=project_graph, - python_graph=python_graph, - ) - ) - return atoms - if parsed.kind == SelectorKind.TEST: - raise unit_test_selector_error(selector=raw) - if parsed.kind == SelectorKind.PATH: - atoms = _resolve_path(raw=raw, project_graph=project_graph, python_graph=python_graph) - if parsed.upstream: - atoms = atoms | frozenset( - _required_terminal_loader_atoms( - selected_atoms=set(atoms), - project_graph=project_graph, - python_graph=python_graph, - ) - ) - return atoms - if parsed.kind in _SQL_SELECTOR_KINDS: - atoms = _resolve_sql(raw=raw, project_graph=project_graph) - if parsed.upstream: - atoms = atoms | frozenset( - _required_terminal_loader_atoms( - selected_atoms=set(atoms), - project_graph=project_graph, - python_graph=python_graph, - ) - ) - return atoms - if parsed.kind in _PYTHON_SELECTOR_KINDS: - terminal_loader_by_name: dict[str, str] = _terminal_loader_by_name( - project_graph=project_graph - ) - matched_terminal_loaders: tuple[str, ...] = tuple( - sorted(name for name in terminal_loader_by_name if fnmatchcase(name, parsed.value)) - ) - if parsed.kind == SelectorKind.LOADER and matched_terminal_loaders: - loader_name: str = matched_terminal_loaders[0] - source_name: str = terminal_loader_by_name[loader_name] - raise PlannerInputError( - f"loader selector '{parsed.value}' matches managed source loader " - f"'{loader_name}'; select it as source:{source_name}", - code="S007", - ) - return _resolve_python(raw=raw, python_graph=python_graph) - if parsed.kind == SelectorKind.TAG: - atoms = _resolve_tag(raw=raw, project_graph=project_graph, python_graph=python_graph) - if parsed.upstream: - atoms = atoms | frozenset( - _required_terminal_loader_atoms( - selected_atoms=set(atoms), - project_graph=project_graph, - python_graph=python_graph, - ) - ) - return atoms - if parsed.kind == SelectorKind.NAME: - atoms = _resolve_name( - raw=raw, parsed=parsed, project_graph=project_graph, python_graph=python_graph - ) - if parsed.upstream: - atoms = atoms | frozenset( - _required_terminal_loader_atoms( - selected_atoms=set(atoms), - project_graph=project_graph, - python_graph=python_graph, - ) - ) - return atoms - raise PlannerInputError(f"unsupported selector '{raw}'") - - -def _resolve_sql(*, raw: str, project_graph: ProjectGraph) -> frozenset[_SelectionAtom]: - return frozenset( - _sql_atom(key) - for key in select_project_graph(graph=project_graph, select=(raw,), exclude=()) - ) - - -def _resolve_python(*, raw: str, python_graph: PythonNodeGraph) -> frozenset[_SelectionAtom]: - return frozenset( - _python_atom(name) - for name in resolve_python_node_selectors(select=(raw,), exclude=(), graph=python_graph) - ) - - -def _resolve_tag( - *, raw: str, project_graph: ProjectGraph, python_graph: PythonNodeGraph -) -> frozenset[_SelectionAtom]: - atoms: set[_SelectionAtom] = set() - try: - atoms.update(_resolve_sql(raw=raw, project_graph=project_graph)) - except PlannerInputError as error: - if error.code != TAG_NOT_FOUND_ERROR_CODE: - raise - try: - atoms.update(_resolve_python(raw=raw, python_graph=python_graph)) - except PlannerInputError as error: - if error.code != TAG_NOT_FOUND_ERROR_CODE: - raise - if not atoms: - parsed: ParsedSelector | PathSelector = parse_project_selector(raw) - tag_value: str = parsed.value if isinstance(parsed, ParsedSelector) else raw - raise PlannerInputError(f"no SQL resources or Python nodes found with tag '{tag_value}'") - return frozenset(atoms) - - -def _resolve_path( - *, raw: str, project_graph: ProjectGraph, python_graph: PythonNodeGraph -) -> frozenset[_SelectionAtom]: - parsed: ParsedSelector | PathSelector = parse_project_selector(raw) - if not isinstance(parsed, ParsedSelector): - raise PlannerInputError(f"unsupported path selector '{raw}'") - folder: str = parsed.value.replace("\\", "/").strip("/") - root: str = folder.split("/", 1)[0] - if root == SQL_MODEL_PATH_ROOT: - return _resolve_sql(raw=raw, project_graph=project_graph) - if root in PYTHON_NODE_PATH_ROOTS: - return _resolve_python(raw=raw, python_graph=python_graph) - raise PlannerInputError( - PATH_SELECTOR_EXPLICIT_ROOT_ERROR, - code="S012", - ) - - -def _resolve_name( - *, - raw: str, - parsed: ParsedSelector, - project_graph: ProjectGraph, - python_graph: PythonNodeGraph, -) -> frozenset[_SelectionAtom]: - if not _is_name_pattern(parsed.value): - sql_key: CompiledObjectKey | None = project_graph.all_keys.get(parsed.value) - python_exists: bool = parsed.value in python_graph.nodes_by_name - terminal_loader_names: frozenset[str] = frozenset( - _terminal_loader_by_name(project_graph=project_graph) - ) - if sql_key is not None and parsed.value in terminal_loader_names: - return _resolve_sql(raw=raw, project_graph=project_graph) - if sql_key is not None and python_exists: - raise PlannerInputError( - f"selector name '{parsed.value}' matches both a SQL resource and a Python node; " - "resource names must be globally unique" - ) - if sql_key is not None: - return _resolve_sql(raw=raw, project_graph=project_graph) - if python_exists: - return _resolve_python(raw=raw, python_graph=python_graph) - raise PlannerInputError(f"unknown selector name '{parsed.value}'", code="S007") - - terminal_loader_names: frozenset[str] = frozenset( - _terminal_loader_by_name(project_graph=project_graph) - ) - sql_names: frozenset[str] = frozenset( - name for name in project_graph.all_keys if fnmatchcase(name, parsed.value) - ) - python_names: frozenset[str] = frozenset( - name for name in python_graph.nodes_by_name if fnmatchcase(name, parsed.value) - ) - conflicting_names: frozenset[str] = (sql_names & python_names) - terminal_loader_names - if conflicting_names: - conflicting_name: str = sorted(conflicting_names)[0] - raise PlannerInputError( - f"selector name '{conflicting_name}' matches both a SQL resource and a Python node; " - "resource names must be globally unique" - ) - atoms: set[_SelectionAtom] = set() - if sql_names: - atoms.update(_resolve_sql(raw=raw, project_graph=project_graph)) - if python_names: - python_atoms: frozenset[_SelectionAtom] = _resolve_python( - raw=raw, python_graph=python_graph - ) - duplicate_terminal_atoms: set[_SelectionAtom] = { - _python_atom(name) for name in sql_names & terminal_loader_names - } - atoms.update(python_atoms - duplicate_terminal_atoms) - if atoms: - return frozenset(atoms) - raise PlannerInputError(f"unknown selector name '{parsed.value}'", code="S007") - - -def _is_name_pattern(value: str) -> bool: - return any(character in value for character in "*?[") - - -def _build_selection(atoms: set[_SelectionAtom]) -> PythonSqlSelection: - return PythonSqlSelection( - sql_keys=frozenset( - atom.value for atom in atoms if isinstance(atom.value, CompiledObjectKey) - ), - python_node_names=frozenset(atom.value for atom in atoms if isinstance(atom.value, str)), - ) - - -def _required_sql_resource_atoms( - *, selected_atoms: set[_SelectionAtom], project_graph: ProjectGraph -) -> set[_SelectionAtom]: - selected_sql_keys: frozenset[CompiledObjectKey] = frozenset( - atom.value for atom in selected_atoms if isinstance(atom.value, CompiledObjectKey) - ) - return { - _sql_atom(key) - for key in expand_build_resource_selection( - selected_keys=selected_sql_keys, - upstream=project_graph.upstream_deps, - downstream=project_graph.downstream_deps, - include_upstream_functions=True, - include_upstream_seeds=False, - include_downstream_functions=False, - ) - } - - -def _required_terminal_loader_atoms( - *, - selected_atoms: set[_SelectionAtom], - project_graph: ProjectGraph, - python_graph: PythonNodeGraph, -) -> set[_SelectionAtom]: - source_loader_by_name: dict[str, str] = _source_loader_by_name(project_graph=project_graph) - required: set[_SelectionAtom] = set() - atom: _SelectionAtom - for atom in selected_atoms: - if not isinstance(atom.value, CompiledObjectKey): - continue - if atom.value.resource_type != CompiledResourceType.SOURCE: - continue - loader_name: str | None = source_loader_by_name.get(atom.value.name) - if loader_name is None: - continue - if loader_name == atom.value.name: - if loader_name in python_graph.nodes_by_name: - required.add(_python_atom(loader_name)) - required.update( - _python_atom(name) - for name in transitive_closure( - start=loader_name, edges=python_graph.upstream_deps - ) - ) - continue - if loader_name in python_graph.nodes_by_name: - required.add(_python_atom(loader_name)) - required.update( - _python_atom(name) - for name in transitive_closure(start=loader_name, edges=python_graph.upstream_deps) - ) - return required - - -def _validate_selected_dependencies( - *, - selected_atoms: set[_SelectionAtom], - project_graph: ProjectGraph, - python_graph: PythonNodeGraph, -) -> None: - selected_python_names: frozenset[str] = frozenset( - atom.value for atom in selected_atoms if isinstance(atom.value, str) - ) - source_loader_by_name: dict[str, str] = _source_loader_by_name(project_graph=project_graph) - atom: _SelectionAtom - for atom in selected_atoms: - if not isinstance(atom.value, CompiledObjectKey): - continue - if atom.value.resource_type != CompiledResourceType.SOURCE: - continue - loader_name: str | None = source_loader_by_name.get(atom.value.name) - if loader_name is None or loader_name == atom.value.name: - continue - if loader_name in selected_python_names: - continue - raise PlannerInputError( - f"Source '{atom.value.name}' requires loader '{loader_name}', but that loader was " - "not selected; select it directly or use upstream expansion" - ) - node_name: str - for node_name in selected_python_names: - upstream_name: str - for upstream_name in python_graph.upstream_deps.get(node_name, ()): - if upstream_name in selected_python_names: - continue - raise PlannerInputError( - f"Python node '{node_name}' depends on unselected Python node " - f"'{upstream_name}'; select it directly or use upstream expansion" - ) - - -def _source_loader_by_name(*, project_graph: ProjectGraph) -> dict[str, str]: - return { - source.name: source.source_entry.loader - for source in project_graph.project.sources - if source.source_entry.loader is not None - } - - -def _terminal_loader_by_name(*, project_graph: ProjectGraph) -> dict[str, str]: - return { - loader_name: source_name - for source_name, loader_name in _source_loader_by_name(project_graph=project_graph).items() - } - - -def _sql_atom(key: CompiledObjectKey) -> _SelectionAtom: - return _SelectionAtom(kind="sql", value=key) - - -def _python_atom(name: str) -> _SelectionAtom: - return _SelectionAtom(kind="python", value=name) diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py b/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py similarity index 78% rename from src/sqlbuild/compiler/python_nodes/_helpers/inventory.py rename to src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py index 5b7b95f5ac..9a94fa2409 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py @@ -1,4 +1,4 @@ -"""Inventory helpers for executable Python DAG nodes.""" +"""Python-node inventory over loaded user functions: signatures, providers and dependencies.""" from __future__ import annotations @@ -8,6 +8,7 @@ from pathlib import Path from typing import Protocol, get_type_hints +from sqlbuild._native import python_node_graph_indexes from sqlbuild.compiler.discovery.models import ( DiscoveredAssetFunction, DiscoveredCheckFunction, @@ -52,18 +53,24 @@ def build_python_node_graph(*, discovered_inputs: DiscoveredProjectInputs) -> Py check_functions=discovered_inputs.check_functions, providers=discovered_inputs.providers, ) - dependency_edges: tuple[PythonNodeDependencyEdge, ...] = build_python_node_dependency_edges( - nodes=nodes + edges, upstream, downstream, tags, paths = python_node_graph_indexes( + [ + (node.name, node.tags, node.relative_path.as_posix(), upstream_names) + for node, upstream_names in zip(nodes, _upstream_node_names(nodes), strict=True) + ] ) return PythonNodeGraph( nodes=nodes, - dependency_edges=dependency_edges, - upstream_deps=build_python_node_upstream_deps(nodes=nodes, edges=dependency_edges), - downstream_deps=build_python_node_downstream_deps(nodes=nodes, edges=dependency_edges), - tag_index=build_python_node_tag_index(nodes=nodes), - path_index=build_python_node_path_index(nodes=nodes), + dependency_edges=tuple( + PythonNodeDependencyEdge(upstream_name=upstream_name, downstream_name=downstream_name) + for upstream_name, downstream_name in edges + ), + upstream_deps={name: tuple(names) for name, names in upstream}, + downstream_deps={name: tuple(names) for name, names in downstream}, + tag_index={tag: frozenset(names) for tag, names in tags}, + path_index=dict(paths), nodes_by_name={node.name: node for node in nodes}, - nodes_by_typed_selector={_typed_selector(node): node for node in nodes}, + nodes_by_typed_selector={f"{node.kind.value}:{node.name}": node for node in nodes}, ) @@ -96,10 +103,8 @@ def build_python_nodes( return tuple(nodes) -def build_python_node_dependency_edges( - *, nodes: tuple[DiscoveredPythonNode, ...] -) -> tuple[PythonNodeDependencyEdge, ...]: - """Return dependency edges across loaders, tasks, assets, and checks.""" +def _upstream_node_names(nodes: tuple[DiscoveredPythonNode, ...]) -> list[list[str]]: + """Each node's Python-node dependencies, resolving live callables to node names.""" node_by_dependency_key: dict[object | tuple[str, str], DiscoveredPythonNode] = { node.function: node for node in nodes @@ -107,9 +112,9 @@ def build_python_node_dependency_edges( node: DiscoveredPythonNode for node in nodes: node_by_dependency_key[("name", node.name)] = node - - edges: list[PythonNodeDependencyEdge] = [] + upstream_names: list[list[str]] = [] for node in nodes: + names: list[str] = [] dependency: Callable[..., object] | SqlResourceRef for dependency in node.depends_on: if isinstance(dependency, SqlResourceRef): @@ -117,61 +122,10 @@ def build_python_node_dependency_edges( upstream_node: DiscoveredPythonNode | None = node_by_dependency_key.get( _python_node_dependency_key(dependency) ) - if upstream_node is None: - continue - edges.append( - PythonNodeDependencyEdge( - upstream_name=upstream_node.name, - downstream_name=node.name, - ) - ) - return tuple(edges) - - -def build_python_node_upstream_deps( - *, nodes: tuple[DiscoveredPythonNode, ...], edges: tuple[PythonNodeDependencyEdge, ...] -) -> dict[str, tuple[str, ...]]: - """Return upstream Python-node dependency names keyed by downstream node name.""" - - upstream: dict[str, list[str]] = {node.name: [] for node in nodes} - edge: PythonNodeDependencyEdge - for edge in edges: - upstream.setdefault(edge.downstream_name, []).append(edge.upstream_name) - upstream.setdefault(edge.upstream_name, []) - return {name: tuple(values) for name, values in upstream.items()} - - -def build_python_node_downstream_deps( - *, nodes: tuple[DiscoveredPythonNode, ...], edges: tuple[PythonNodeDependencyEdge, ...] -) -> dict[str, tuple[str, ...]]: - """Return downstream Python-node dependency names keyed by upstream node name.""" - - downstream: dict[str, list[str]] = {node.name: [] for node in nodes} - edge: PythonNodeDependencyEdge - for edge in edges: - downstream.setdefault(edge.upstream_name, []).append(edge.downstream_name) - downstream.setdefault(edge.downstream_name, []) - return {name: tuple(values) for name, values in downstream.items()} - - -def build_python_node_tag_index( - *, nodes: tuple[DiscoveredPythonNode, ...] -) -> dict[str, frozenset[str]]: - """Build a tag-to-node-name lookup for tagged Python nodes.""" - - index: dict[str, set[str]] = {} - node: DiscoveredPythonNode - for node in nodes: - tag: str - for tag in node.tags: - index.setdefault(tag, set()).add(node.name) - return {tag: frozenset(names) for tag, names in index.items()} - - -def build_python_node_path_index(*, nodes: tuple[DiscoveredPythonNode, ...]) -> dict[str, str]: - """Build a node-name-to-folder lookup from discovered Python node relative paths.""" - - return {node.name: node.relative_path.parent.as_posix() for node in nodes} + if upstream_node is not None: + names.append(upstream_node.name) + upstream_names.append(names) + return upstream_names class _DiscoveredPythonFunction(Protocol): @@ -397,7 +351,3 @@ def _sql_deps( return tuple( dependency for dependency in dependencies if isinstance(dependency, SqlResourceRef) ) - - -def _typed_selector(node: DiscoveredPythonNode) -> str: - return f"{node.kind.value}:{node.name}" diff --git a/src/sqlbuild/compiler/python_nodes/constants.py b/src/sqlbuild/compiler/python_nodes/constants.py deleted file mode 100644 index 80b0c465e9..0000000000 --- a/src/sqlbuild/compiler/python_nodes/constants.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Python-node selector constants.""" - -from __future__ import annotations - -EMPTY_SELECTOR_FOLDER: str = "" -SQL_MODEL_PATH_ROOT: str = "models" -PYTHON_NODE_PATH_ROOTS: frozenset[str] = frozenset({"python"}) -UNIFIED_PATH_ROOTS: frozenset[str] = frozenset({*PYTHON_NODE_PATH_ROOTS, SQL_MODEL_PATH_ROOT}) -TAG_NOT_FOUND_ERROR_CODE: str = "S008" diff --git a/src/sqlbuild/compiler/python_nodes/main/_graph.py b/src/sqlbuild/compiler/python_nodes/main/_graph.py index 5386e78317..b968382904 100644 --- a/src/sqlbuild/compiler/python_nodes/main/_graph.py +++ b/src/sqlbuild/compiler/python_nodes/main/_graph.py @@ -3,7 +3,7 @@ from __future__ import annotations from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.python_nodes._helpers.inventory import build_python_node_graph +from sqlbuild.compiler.python_nodes._helpers.user_inventory import build_python_node_graph from sqlbuild.compiler.python_nodes.models import PythonNodeGraph diff --git a/src/sqlbuild/compiler/python_nodes/main/_run_selection.py b/src/sqlbuild/compiler/python_nodes/main/_run_selection.py index a38850eea1..c78145aebb 100644 --- a/src/sqlbuild/compiler/python_nodes/main/_run_selection.py +++ b/src/sqlbuild/compiler/python_nodes/main/_run_selection.py @@ -2,14 +2,11 @@ from __future__ import annotations +from sqlbuild._native import compiled_graph_keys, resolve_python_sql_run_selection from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes._helpers.inventory import build_python_node_graph -from sqlbuild.compiler.python_nodes._helpers.run_selectors import resolve_python_sql_run_selectors -from sqlbuild.compiler.python_nodes.models import ( - PythonNodeGraph, - PythonSqlRunSelection, -) +from sqlbuild.compiler.python_nodes._helpers.user_inventory import build_python_node_graph +from sqlbuild.compiler.python_nodes.models import PythonSqlRunSelection def resolve_python_sql_run_selection_from_inputs( @@ -21,10 +18,12 @@ def resolve_python_sql_run_selection_from_inputs( ) -> PythonSqlRunSelection: """Resolve run selectors using discovered Python-node project inputs.""" - python_graph: PythonNodeGraph = build_python_node_graph(discovered_inputs=discovered_inputs) - return resolve_python_sql_run_selectors( + sql_keys, python_node_names = resolve_python_sql_run_selection( + project_graph=project_graph, + python_graph=build_python_node_graph(discovered_inputs=discovered_inputs), select=select, exclude=exclude, - project_graph=project_graph, - python_graph=python_graph, + ) + return PythonSqlRunSelection( + sql_keys=compiled_graph_keys(sql_keys), python_node_names=frozenset(python_node_names) ) diff --git a/src/sqlbuild/compiler/python_nodes/main/selectors.py b/src/sqlbuild/compiler/python_nodes/main/selectors.py deleted file mode 100644 index ffe941f710..0000000000 --- a/src/sqlbuild/compiler/python_nodes/main/selectors.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public Python-node selector entrypoints.""" - -from __future__ import annotations - -from sqlbuild.compiler.python_nodes._helpers.selectors import resolve_python_node_selectors -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph - - -def resolve_python_nodes_from_selectors( - *, select: tuple[str, ...], exclude: tuple[str, ...], graph: PythonNodeGraph -) -> frozenset[str]: - """Resolve raw selector strings into Python-node names.""" - - return resolve_python_node_selectors(select=select, exclude=exclude, graph=graph) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py index 848edbf402..d47675a819 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py @@ -10,6 +10,7 @@ from types import MappingProxyType, ModuleType from typing import Any, cast +from sqlbuild._native import compiled_graph_keys, resolve_python_sql_run_selection from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -34,8 +35,8 @@ ) from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes._helpers.inventory import build_python_node_graph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph +from sqlbuild.compiler.python_nodes._helpers.user_inventory import build_python_node_graph +from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunSelection from sqlbuild.providers import Provider from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity @@ -570,3 +571,18 @@ def build_model_depends_on_intermediate_loader_project_graph() -> ProjectGraph: path_index=project_graph.path_index, all_keys=project_graph.all_keys, ) + + +def resolve_run_selection( + *, + select: tuple[str, ...], + exclude: tuple[str, ...], + project_graph: ProjectGraph, + python_graph: PythonNodeGraph, +) -> PythonSqlRunSelection: + sql_keys, python_node_names = resolve_python_sql_run_selection( + project_graph=project_graph, python_graph=python_graph, select=select, exclude=exclude + ) + return PythonSqlRunSelection( + sql_keys=compiled_graph_keys(sql_keys), python_node_names=frozenset(python_node_names) + ) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py index 0d7d26d11f..5bc5f99136 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py @@ -18,7 +18,7 @@ DiscoveredTaskFunction, ) from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.python_nodes._helpers.inventory import ( +from sqlbuild.compiler.python_nodes._helpers.user_inventory import ( build_python_node_graph, build_python_nodes, ) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py index f393b6c4b2..a34cd53d4b 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py @@ -5,7 +5,6 @@ import pytest from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes._helpers.run_selectors import resolve_python_sql_run_selectors from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunSelection from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonSqlSelectorErrorTestCase, @@ -14,6 +13,7 @@ from tests.unit.src.sqlbuild.compiler.python_nodes._helpers.helpers import ( build_orders_project_graph, build_orders_python_node_graph, + resolve_run_selection, ) @@ -52,7 +52,7 @@ def test_given_run_selectors_when_resolving_then_excludes_python_checks( project_graph: ProjectGraph = build_orders_project_graph() python_graph: PythonNodeGraph = build_orders_python_node_graph() - result: PythonSqlRunSelection = resolve_python_sql_run_selectors( + result: PythonSqlRunSelection = resolve_run_selection( select=test_case.select, exclude=test_case.exclude, project_graph=project_graph, @@ -99,7 +99,7 @@ def test_given_run_selector_selects_check_when_resolving_then_raises( python_graph: PythonNodeGraph = build_orders_python_node_graph() with pytest.raises(test_case.expected_error_type, match=test_case.expected_error_fragment): - resolve_python_sql_run_selectors( + resolve_run_selection( select=test_case.select, exclude=test_case.exclude, project_graph=project_graph, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py index 3a9d8b4aa0..2bd507852b 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py @@ -4,10 +4,10 @@ import pytest +from sqlbuild._native import resolve_python_node_selection from sqlbuild.compiler.planner.main.selection.selector_parse import parse_project_selector from sqlbuild.compiler.planner.models import ParsedSelector from sqlbuild.compiler.planner.types import SelectorKind -from sqlbuild.compiler.python_nodes._helpers.selectors import resolve_python_node_selectors from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonNodeParseSelectorTestCase, @@ -181,7 +181,7 @@ def test_given_python_node_selectors_when_resolving_then_returns_expected_names( ) -> None: graph: PythonNodeGraph = build_orders_python_node_graph() - result: frozenset[str] = resolve_python_node_selectors( + result: frozenset[str] = resolve_python_node_selection( select=test_case.select, exclude=test_case.exclude, graph=graph, @@ -244,7 +244,7 @@ def test_given_invalid_python_node_selector_when_resolving_then_raises_clear_err graph: PythonNodeGraph = build_orders_python_node_graph() with pytest.raises(test_case.expected_error_type, match=test_case.expected_error_fragment): - resolve_python_node_selectors( + resolve_python_node_selection( select=test_case.select, exclude=test_case.exclude, graph=graph, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py index 33d4ac2e18..888a3350c6 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py @@ -7,11 +7,7 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors -from sqlbuild.compiler.python_nodes._helpers.unified_selectors import ( - resolve_python_sql_selectors, - validate_python_sql_boundaries, -) -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlSelection +from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunSelection from sqlbuild.refs import seed from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonSqlSelectorErrorTestCase, @@ -28,6 +24,7 @@ build_sql_ref_python_node_graph, build_terminal_loader_python_node_graph, model_ref, + resolve_run_selection, source_ref, ) @@ -113,10 +110,10 @@ expected_python_node_names=frozenset(), ), PythonSqlSelectorTestCase( - description="selects source and intermediate loader without duplicate source names", + description="selects source and intermediate loader as its own source", select=("+source:raw_orders +loader:load_events",), exclude=(), - expected_sql_names=frozenset({"raw_orders"}), + expected_sql_names=frozenset({"raw_orders", "load_events"}), expected_python_node_names=frozenset({"prepare_orders", "load_events"}), ), PythonSqlSelectorTestCase( @@ -135,7 +132,7 @@ def test_given_unified_selectors_when_resolving_then_returns_sql_and_python_sele project_graph: ProjectGraph = build_orders_project_graph() python_graph: PythonNodeGraph = build_orders_python_node_graph() - result: PythonSqlSelection = resolve_python_sql_selectors( + result: PythonSqlRunSelection = resolve_run_selection( select=test_case.select, exclude=test_case.exclude, project_graph=project_graph, @@ -180,7 +177,7 @@ def test_given_unknown_unified_selector_when_resolving_then_raises_clear_error( python_graph: PythonNodeGraph = build_orders_python_node_graph() with pytest.raises(test_case.expected_error_type, match=test_case.expected_error_fragment): - resolve_python_sql_selectors( + resolve_run_selection( select=test_case.select, exclude=test_case.exclude, project_graph=project_graph, @@ -208,7 +205,7 @@ def test_given_managed_source_loader_glob_when_resolving_then_requires_source_se python_graph: PythonNodeGraph = build_terminal_loader_python_node_graph() with pytest.raises(test_case.expected_error_type, match=test_case.expected_error_fragment): - resolve_python_sql_selectors( + resolve_run_selection( select=test_case.select, exclude=test_case.exclude, project_graph=project_graph, @@ -246,7 +243,7 @@ def test_given_managed_source_loader_glob_when_resolving_then_requires_source_se def test_given_python_sql_dependency_when_expanding_selectors_then_crosses_the_boundary( test_case: PythonSqlSelectorTestCase, ) -> None: - result: PythonSqlSelection = resolve_python_sql_selectors( + result: PythonSqlRunSelection = resolve_run_selection( select=test_case.select, exclude=test_case.exclude, project_graph=build_orders_project_graph(), @@ -294,12 +291,11 @@ def test_given_python_sql_dependency_when_expanding_selectors_then_crosses_the_b def test_given_sibling_python_nodes_when_expanding_both_ways_then_starts_from_the_roots( test_case: PythonSqlSelectorTestCase, ) -> None: - result: PythonSqlSelection = resolve_python_sql_selectors( + result: PythonSqlRunSelection = resolve_run_selection( select=test_case.select, exclude=test_case.exclude, project_graph=build_orders_project_graph(), python_graph=build_sibling_sql_ref_python_node_graph(), - validate_dependencies=False, ) assert frozenset(key.name for key in result.sql_keys) == test_case.expected_sql_names @@ -352,12 +348,11 @@ def test_given_sql_only_selector_when_resolving_unified_then_sql_keys_match_sql_ ) -> None: project_graph: ProjectGraph = build_orders_project_graph() - result: PythonSqlSelection = resolve_python_sql_selectors( + result: PythonSqlRunSelection = resolve_run_selection( select=(test_case.selector,), exclude=(), project_graph=project_graph, python_graph=build_sibling_sql_ref_python_node_graph(), - validate_dependencies=False, ) sql_only: frozenset[CompiledObjectKey] = resolve_project_selectors( select=(test_case.selector,), @@ -421,7 +416,7 @@ def test_given_invalid_typed_sql_ref_when_resolving_then_raises_clear_error( ) with pytest.raises(test_case.expected_error_type, match=test_case.expected_error_fragment): - resolve_python_sql_selectors( + resolve_run_selection( select=test_case.select, exclude=test_case.exclude, project_graph=project_graph, @@ -475,7 +470,7 @@ def test_given_invalid_terminal_loader_dependency_when_validating_boundaries_the python_graph: PythonNodeGraph = build_python_node_graph_for_case(test_case.python_graph_case) with pytest.raises(test_case.expected_error_type, match=test_case.expected_error_fragment): - resolve_python_sql_selectors( + resolve_run_selection( select=test_case.select, exclude=test_case.exclude, project_graph=project_graph, @@ -507,7 +502,7 @@ def test_given_invalid_sql_model_dependency_when_validating_boundaries_then_rais python_graph: PythonNodeGraph = build_python_node_graph_for_case(test_case.python_graph_case) with pytest.raises(test_case.expected_error_type, match=test_case.expected_error_fragment): - resolve_python_sql_selectors( + resolve_run_selection( select=test_case.select, exclude=test_case.exclude, project_graph=project_graph, @@ -537,7 +532,7 @@ def test_given_sql_downstream_task_feeds_loader_when_validating_then_raises( python_graph: PythonNodeGraph = build_sql_downstream_task_to_loader_python_node_graph() with pytest.raises(test_case.expected_error_type, match=test_case.expected_error_fragment): - resolve_python_sql_selectors( + resolve_run_selection( select=test_case.select, exclude=test_case.exclude, project_graph=project_graph, @@ -581,5 +576,9 @@ def test_given_intermediate_loader_dependency_when_validating_boundaries_then_it project_graph: ProjectGraph = build_orders_project_graph() python_graph: PythonNodeGraph = build_python_node_graph_for_case(test_case.python_graph_case) - validate_python_sql_boundaries(project_graph=project_graph, python_graph=python_graph) + result: PythonSqlRunSelection = resolve_run_selection( + select=(), exclude=(), project_graph=project_graph, python_graph=python_graph + ) + assert test_case.expected_error_fragment is None + assert result.python_node_names diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index 1b7ba331ab..c8d2523d65 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -22,7 +22,7 @@ DiscoveredProjectInputs, DiscoveredTaskFunction, ) -from sqlbuild.compiler.python_nodes._helpers.inventory import build_python_node_graph +from sqlbuild.compiler.python_nodes._helpers.user_inventory import build_python_node_graph from sqlbuild.compiler.python_nodes._helpers.run_lifecycle import ( build_python_sql_run_lifecycle_plan, ) From b6d02785db1a8c21f5e8e9726916d39839524b1c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:26:02 +0100 Subject: [PATCH 17/24] style: sort imports in the Python-node scheduler test helpers --- .../unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index c8d2523d65..5b38105589 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -22,10 +22,10 @@ DiscoveredProjectInputs, DiscoveredTaskFunction, ) -from sqlbuild.compiler.python_nodes._helpers.user_inventory import build_python_node_graph from sqlbuild.compiler.python_nodes._helpers.run_lifecycle import ( build_python_sql_run_lifecycle_plan, ) +from sqlbuild.compiler.python_nodes._helpers.user_inventory import build_python_node_graph from sqlbuild.compiler.python_nodes.models import ( DiscoveredPythonNode, PythonNodeGraph, From 2e9c506818cdb14e7252920ef2fbc99a34f7cd93 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:38:55 +0100 Subject: [PATCH 18/24] refactor: split user macro loading and the macro bridge from native expansion Project macro module loading and the context mappings user macros read move to render/user_macros.py, and the macro bridge and its render-stage runner to macro_bridge/classes/macro_bridge.py, both kept as user-code. Macro expansion now returns MacroExpansionResult natively, the expansion class table is built natively, and find_macro_call_names replaces the raw native call-name scan. --- .../compile_attachment/model_inputs.rs | 7 +- .../_helpers/compile_driver/assembly.rs | 2 +- .../compile_driver/assembly_checks.rs | 7 +- .../compile_driver/assembly_models.rs | 14 +- .../_helpers/compile_driver/inputs.rs | 2 +- .../_helpers/frontier/compile_frontier.rs | 2 +- .../bindings/_helpers/render/macro_calls.rs | 43 ++- .../_helpers/render/macro_expansion_result.rs | 320 ++++++++++++++++++ .../src/bindings/_helpers/render/mod.rs | 1 + .../bindings/_helpers/render/registration.rs | 3 +- python-boundary.toml | 12 +- scripts/fensu_policy/constants.py | 2 +- src/sqlbuild/_native.pyi | 14 +- .../compile/_helpers/render/sql_vars.py | 12 +- .../render/{macros.py => user_macros.py} | 203 +---------- .../compile/main/_static_macro_inventory.py | 2 +- .../compile/main/sql_expansion_context.py | 2 +- .../macro_bridge/classes/macro_bridge.py | 35 +- .../main/run_with_macro_bridge.py | 35 -- .../sqlbuild/cli/commands/main/test_rules.py | 2 +- .../sqlbuild/compiler/attachments/helpers.py | 3 +- .../src/sqlbuild/compiler/compile/helpers.py | 4 +- .../src/sqlbuild/compiler/scopes/helpers.py | 2 +- .../rules/test_canonical_operations.py | 2 +- .../compiler/compile/_helpers/helpers.py | 8 +- .../compiler/compile/_helpers/test_macros.py | 11 +- .../_helpers/test_rebind_declaration_scope.py | 2 +- .../main/test_run_with_macro_bridge.py | 2 +- .../_helpers/sql_test_assembly/helpers.py | 4 +- 29 files changed, 455 insertions(+), 303 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs rename src/sqlbuild/compiler/compile/_helpers/render/{macros.py => user_macros.py} (71%) delete mode 100644 src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index b891d4557f..e208d7f7bb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -9,6 +9,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::compiled_project::loop_classes::model_loop_classes; +use crate::bindings::_helpers::render::macro_expansion_result::macro_expansion_classes; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; @@ -22,7 +23,6 @@ const VALIDATION_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.val const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; const CURSOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; -const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.macros"; const TEMPLATING_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.templating"; const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; const PROJECT_VARS_MODULE: &str = "sqlbuild._native"; @@ -108,10 +108,7 @@ fn macro_inputs<'py>(context: &Bound<'py, PyAny>) -> PyResult "macro_argument_error", imported(py, COMPILE_ERRORS_MODULE, "MacroArgumentError")?, ), - ( - "expansion", - imported(py, MACROS_MODULE, "macro_expansion_classes")?.call0()?, - ), + ("expansion", macro_expansion_classes(py)?.into_any()), ( "generic_syntax", imported(py, SQL_ANALYSIS_MODELS_MODULE, "SqlLexicalSyntax")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs index 807c9a2003..635fbd2cd9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -16,7 +16,7 @@ const RECORD_ANALYSES_MODULE: &str = "sqlbuild._native"; const EXPLICIT_REFERENCES_MODULE: &str = "sqlbuild.compiler.compile._helpers.explicit_references.checks"; const ACTIVE_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; -const RUN_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.run_with_macro_bridge"; +const RUN_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.classes.macro_bridge"; const OPTIONAL: [&str; 5] = [ "inference_profile", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs index dd0c9ae3ab..87816897fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs @@ -9,7 +9,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; -const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.macros"; +const NATIVE_MODULE: &str = "sqlbuild._native"; const CURSOR_INTRINSICS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; /// The `CompiledSqlScenario` fields copied from its input, in Python's keyword order. @@ -73,7 +73,7 @@ pub(crate) fn test_model_query_overrides<'py>( ("lookup", lookup), ], )?; - let expand: Bound<'py, PyAny> = imported(py, MACROS_MODULE, "expand_sql_macros")?; + let expand: Bound<'py, PyAny> = py.import(NATIVE_MODULE)?.getattr("expand_sql_macros")?; let intrinsics: Bound<'py, PyAny> = imported( py, CURSOR_INTRINSICS_MODULE, @@ -100,7 +100,8 @@ pub(crate) fn test_model_query_overrides<'py>( ("macro_context", macro_context.clone()), ("declaration_resolver", declaration_resolver.clone()), ], - )?; + )? + .getattr("sql")?; let query: Bound<'py, PyAny> = call_with( &intrinsics, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index d85ab069e8..0e23885df8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -9,11 +9,11 @@ use crate::bindings::_helpers::compile_driver::assembly_analysis::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::hand_built::compiled_model::compiled_model as hand_built_model; +use crate::bindings::_helpers::render::macro_calls::top_level_macro_call_names; const NATIVE_MODULE: &str = "sqlbuild._native"; const CURSOR_INTRINSICS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; -const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.macros"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; @@ -24,7 +24,6 @@ struct ModelStages<'py> { placeholders: Bound<'py, PyAny>, validate_hooks: Bound<'py, PyAny>, validate_sql: Bound<'py, PyAny>, - macro_calls: Bound<'py, PyAny>, native_input: Bound<'py, PyAny>, } @@ -53,7 +52,6 @@ pub(crate) fn compiled_models<'py>( placeholders: native.getattr("model_placeholders")?, validate_hooks: native.getattr("validate_hook_sql_syntax")?, validate_sql: native.getattr("validate_sql_syntax")?, - macro_calls: imported(py, MACROS_MODULE, "find_macro_call_names")?, native_input: native.getattr("NativeModelInput")?, }; let analysis_enabled: bool = context.sql_analysis_enabled.is_truthy()?; @@ -158,9 +156,13 @@ fn compiled_model<'py>( let macro_deps: Bound<'py, PyAny> = if declared_macro_deps.is_truthy()? { declared_macro_deps } else { - stages - .macro_calls - .call1((model_input.getattr("macro_source_sql")?,))? + top_level_macro_call_names( + py, + &model_input + .getattr("macro_source_sql")? + .extract::()?, + )? + .into_any() }; let native_model: &Bound<'py, PyAny> = &request.native_model; if model_input.is_instance(&stages.native_input)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index ceb29e8a99..b737562867 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -14,7 +14,7 @@ const DECLARATION_SCOPE_MODULE: &str = "sqlbuild._native"; const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; const CONTEXT_TEMPLATES_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.context_templates"; -const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.macros"; +const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const ATTACH_CHECKS_MODULE: &str = "sqlbuild._native"; const ATTACH_RESOURCES_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs index 0c6f8031a5..01e4bdacb1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::frontier::stage_capture::write_stage_capture; const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; -const RUN_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.run_with_macro_bridge"; +const RUN_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.classes.macro_bridge"; /// Produce the frontier object for `until`, capturing it on request. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs index 348476a538..277833513d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs @@ -2,6 +2,7 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyTuple; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_render::macro_calls::main::macro_call_names::macro_call_names as scan_names; use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites as scan_sites; @@ -9,10 +10,15 @@ use sqlbuild_render::macro_calls::main::splice_macro_calls::splice_macro_calls a use sqlbuild_render::macro_calls::models::{MacroCallScan, MacroScanFailure}; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::render::macro_call_memo::MacroCallMemo; use crate::bindings::_helpers::render::macro_call_rows::{SiteRow, SpanRow, site_row, span_row}; +use crate::bindings::_helpers::render::macro_expansion::raised; use crate::bindings::types::CompilerDetach; +const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const DECLARATION_CALLS: [&str; 2] = ["enum", "const"]; + /// `(call start or None, call name or None, message)` of the error Python's scan raises. type FailureRow = (Option, Option, &'static str); @@ -37,20 +43,33 @@ pub(crate) fn scan_macro_call_sites( )) } -/// Python's `find_macro_call_names` names, or the `CompileInputError` message of its scan. -#[pyfunction] -pub(crate) fn macro_call_names( - py: Python<'_>, +/// Python's `find_macro_call_names`: top-level call names in encounter order, declarations excluded. +pub(crate) fn top_level_macro_call_names<'py>( + py: Python<'py>, sql: &str, - nested: bool, -) -> PyResult<(Option>, Option<&'static str>)> { +) -> PyResult> { let names = py - .compiler_detach(|| Ok(scan_names(sql, nested))) + .compiler_detach(|| Ok(scan_names(sql, false))) .map_err(compiler_error)?; - Ok(match names { - Ok(names) => (Some(names), None), - Err(error) => (None, Some(error.message())), - }) + match names { + Ok(names) => PyTuple::new( + py, + names + .into_iter() + .filter(|name| !DECLARATION_CALLS.contains(&name.as_str())) + .collect::>(), + ), + Err(error) => Err(raised( + &imported(py, COMPILE_EXCEPTIONS_MODULE, "CompileInputError")?, + error.message(), + )), + } +} + +/// Unique authored macro call names in encounter order, declaration calls excluded. +#[pyfunction] +fn find_macro_call_names<'py>(py: Python<'py>, sql: &str) -> PyResult> { + top_level_macro_call_names(py, sql) } /// Splice rendered macro outputs over their call sites, returning the SQL and substitution spans. @@ -71,7 +90,7 @@ pub(crate) fn splice_macro_calls( pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(scan_macro_call_sites, module)?)?; - module.add_function(wrap_pyfunction!(macro_call_names, module)?)?; + module.add_function(wrap_pyfunction!(find_macro_call_names, module)?)?; module.add_function(wrap_pyfunction!(splice_macro_calls, module)?)?; module.add_class::()?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs new file mode 100644 index 0000000000..1137cc93cb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -0,0 +1,320 @@ +//! Expand one authored SQL string's macros into a `MacroExpansionResult`. + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, +}; +use crate::bindings::_helpers::render::macro_call_rows::span_row; +use crate::bindings::_helpers::render::macro_expansion::{ + ExpansionOutput, NativeMacroExpansion, expanded, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const FRONTIER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.frontier.exceptions"; +const SCOPES_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const SCOPES_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const REF_MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; +const REF_TYPES_MODULE: &str = "sqlbuild.python_nodes.types"; +const MACRO_ARGUMENTS_MODULE: &str = + "sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments"; +const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; +const USER_MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; +const ACTIVE_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; +const BRIDGE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.macro_bridge.constants"; +const DIAGNOSTICS_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; +const BUILTINS_MODULE: &str = "builtins"; +const MACRO_TOKEN: &str = "@"; +const GENERATED_ALLOWED: &str = "macro-generated references"; + +static EXPANSION_CLASSES: PyOnceLock> = PyOnceLock::new(); + +/// One expansion's inputs, as Python's `expand_sql_macros_result` takes them. +struct ExpansionRequest<'py> { + sql: Bound<'py, PyString>, + file_path: Bound<'py, PyAny>, + loaded_macros: Bound<'py, PyDict>, + macro_overrides: Option>, + macro_context: Bound<'py, PyAny>, + declaration_resolver: Option>, + declarations: Option>, + consumer: Option>, +} + +fn generated_classes(py: Python<'_>) -> PyResult> { + let native: Bound<'_, PyAny> = py.import(NATIVE_MODULE)?.into_any(); + let help: Bound<'_, PyAny> = native.getattr("explicit_references_help")?.call( + (), + Some(&keywords( + py, + &[("allowed", PyString::new(py, GENERATED_ALLOWED).into_any())], + )?), + )?; + let markers: Bound<'_, PyAny> = imported(py, BUILTINS_MODULE, "list")?.call1((imported( + py, + BRIDGE_CONSTANTS_MODULE, + "GENERATED_REFERENCE_MARKERS", + )?,))?; + keywords( + py, + &[ + ( + "compiler_diagnostic", + imported(py, COMPILE_MODELS_MODULE, "CompilerDiagnostic")?, + ), + ( + "report_diagnostic", + imported(py, DIAGNOSTICS_MODULE, "report_compile_diagnostic")?, + ), + ( + "source_location", + imported(py, CONTRACT_MODELS_MODULE, "SourceLocation")?, + ), + ( + "resource_identity", + imported(py, SCOPES_MODELS_MODULE, "ResourceIdentity")?, + ), + ( + "generic_syntax", + imported(py, SQL_ANALYSIS_MODELS_MODULE, "SqlLexicalSyntax")? + .call0()? + .getattr("native_mapping")?, + ), + ("generated_help", help), + ("all_markers", markers), + ], + ) +} + +fn built_classes(py: Python<'_>) -> PyResult> { + let native: Bound<'_, PyAny> = py.import(NATIVE_MODULE)?.into_any(); + let declaration_kind: Bound<'_, PyAny> = imported(py, SCOPES_TYPES_MODULE, "DeclarationKind")?; + let ref_kinds: Bound<'_, PyAny> = + imported(py, MACRO_ARGUMENTS_MODULE, "typed_reference_kinds")?.call0()?; + let ref_functions: Bound<'_, PyDict> = PyDict::new(py); + for item in ref_kinds.call_method0("items")?.try_iter()? { + let (function, kind): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + ref_functions.set_item(kind.getattr("value")?, function)?; + } + let store_tokens: Bound<'_, PyDict> = keywords( + py, + &[ + ("macro_token", native.getattr("macro_store_token")?), + ("context_token", native.getattr("context_store_token")?), + ], + )?; + keywords( + py, + &[ + ( + "compile_error", + imported(py, COMPILE_EXCEPTIONS_MODULE, "CompileInputError")?, + ), + ( + "macro_argument_error", + imported(py, COMPILE_EXCEPTIONS_MODULE, "MacroArgumentError")?, + ), + ( + "mismatch_error", + imported(py, FRONTIER_EXCEPTIONS_MODULE, "NativeStageMismatchError")?, + ), + ( + "declaration_identity", + imported(py, SCOPES_MODELS_MODULE, "DeclarationIdentity")?, + ), + ("macro_kind", declaration_kind.getattr("MACRO")?), + ("declaration_kind", declaration_kind), + ( + "usage_record", + imported(py, SCOPES_MODELS_MODULE, "UsageRecord")?, + ), + ( + "runtime_usage", + imported(py, SCOPES_TYPES_MODULE, "UsageKind")?.getattr("RUNTIME")?, + ), + ( + "resource_identity", + imported(py, SCOPES_MODELS_MODULE, "ResourceIdentity")?, + ), + ( + "model_kind", + imported(py, SCOPES_TYPES_MODULE, "ResourceKind")?.getattr("MODEL")?, + ), + ( + "resource_ref", + imported(py, REF_MODELS_MODULE, "SqlResourceRef")?, + ), + ("ref_kinds", ref_kinds), + ("ref_functions", ref_functions.into_any()), + ("store_tokens", store_tokens.into_any()), + ( + "relation_renderer", + imported(py, REF_MODELS_MODULE, "RELATION_RENDERER")?, + ), + ( + "invocation_context", + imported(py, USER_MACROS_MODULE, "macro_invocation_context")?, + ), + ("generated", generated_classes(py)?.into_any()), + ("int_type", imported(py, BUILTINS_MODULE, "int")?), + ("float_type", imported(py, BUILTINS_MODULE, "float")?), + ], + ) +} + +/// The classes, enums and helpers the native expansion builds facts and errors from, built once. +pub(crate) fn macro_expansion_classes(py: Python<'_>) -> PyResult> { + EXPANSION_CLASSES + .get_or_try_init(py, || built_classes(py).map(Bound::unbind)) + .map(|classes| classes.bind(py).clone()) +} + +/// `tuple(dict.fromkeys(items))`: the items without repeats, in first-seen order. +fn unique<'py>(py: Python<'py>, items: Vec>) -> PyResult> { + let seen: Bound<'py, PyDict> = PyDict::new(py); + for item in items { + seen.set_item(item, py.None())?; + } + PyTuple::new(py, seen.keys()) +} + +fn result_model<'py>( + py: Python<'py>, + pairs: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with( + &imported(py, COMPILE_MODELS_MODULE, "MacroExpansionResult")?, + pairs, + ) +} + +fn expansion_result<'py>(request: ExpansionRequest<'py>) -> PyResult> { + let py: Python<'py> = request.sql.py(); + if !request.sql.to_str()?.contains(MACRO_TOKEN) { + return result_model(py, &[("sql", request.sql.into_any())]); + } + let declarations: Bound<'py, PyAny> = match (request.declarations, request.declaration_resolver) + { + (Some(declarations), _) => declarations, + (None, Some(resolver)) => call_with( + &imported(py, DECLARATIONS_MODULE, "resolve_declaration_context")?, + &[ + ("resolver", resolver), + ("file_path", request.file_path.clone()), + ], + )?, + (None, None) => py.None().into_bound(py), + }; + let expansion: NativeMacroExpansion = NativeMacroExpansion::new( + &request.file_path, + ( + request.loaded_macros.unbind(), + request + .macro_overrides + .unwrap_or_else(|| PyDict::new(py)) + .unbind(), + request.macro_context.clone().unbind(), + declarations.unbind(), + request.consumer.map_or_else(|| py.None(), Bound::unbind), + ), + request + .macro_context + .getattr("_enforce_explicit_references")? + .is_truthy()?, + macro_expansion_classes(py)?.unbind(), + )?; + let expansion: Bound<'py, NativeMacroExpansion> = Bound::new(py, expansion)?; + let bridge: Bound<'py, PyAny> = + imported(py, ACTIVE_BRIDGE_MODULE, "active_macro_bridge")?.call0()?; + let output: ExpansionOutput = expanded(&expansion, request.sql.to_str()?, &bridge)?; + let span_class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS_MODULE, "ExpansionSpan")?; + let mut spans: Vec> = Vec::with_capacity(output.spans.len()); + for span in output.spans { + let (source_start, source_end, output_start, output_end) = span_row(span); + spans.push(call_with( + &span_class, + &[ + ("source_start", source_start.into_pyobject(py)?.into_any()), + ("source_end", source_end.into_pyobject(py)?.into_any()), + ("output_start", output_start.into_pyobject(py)?.into_any()), + ("output_end", output_end.into_pyobject(py)?.into_any()), + ], + )?); + } + let resource_ref: Bound<'py, PyAny> = imported(py, REF_MODELS_MODULE, "SqlResourceRef")?; + let ref_kind: Bound<'py, PyAny> = imported(py, REF_TYPES_MODULE, "SqlResourceRefKind")?; + let mut references: Vec> = Vec::new(); + for (kind, name) in output.argument_references { + references.push(call_with( + &resource_ref, + &[ + ("kind", ref_kind.call1((kind,))?), + ("name", PyString::new(py, &name).into_any()), + ], + )?); + } + let argument_references: Bound<'py, PyAny> = + imported(py, MACRO_ARGUMENTS_MODULE, "call_site_sql_references")? + .call1((PyTuple::new(py, references)?,))?; + result_model( + py, + &[ + ("sql", PyString::new(py, &output.sql).into_any()), + ("spans", PyTuple::new(py, spans)?.into_any()), + ("argument_references", argument_references), + ("dependencies", unique(py, output.dependencies)?.into_any()), + ("usages", unique(py, output.usages)?.into_any()), + ], + ) +} + +fn optional<'py>(value: Bound<'py, PyAny>) -> Option> { + (!value.is_none()).then_some(value) +} + +/// Expand authored Python macros in one SQL string, keeping substitution spans and facts. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn expand_sql_macros<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "expand_sql_macros", + options, + ( + &["sql", "file_path", "loaded_macros", "macro_context"], + &[ + "macro_overrides", + "declaration_resolver", + "declarations", + "consumer", + ], + ), + )?; + expansion_result(ExpansionRequest { + sql: arguments.value("sql")?.downcast_into()?, + file_path: arguments.value("file_path")?, + loaded_macros: arguments.value("loaded_macros")?.downcast_into()?, + macro_overrides: optional(arguments.value("macro_overrides")?) + .map(Bound::downcast_into) + .transpose()?, + macro_context: arguments.value("macro_context")?, + declaration_resolver: optional(arguments.value("declaration_resolver")?), + declarations: optional(arguments.value("declarations")?), + consumer: optional(arguments.value("consumer")?), + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(expand_sql_macros, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs index 0a81e0ef59..84bb516f80 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs @@ -8,6 +8,7 @@ pub(crate) mod macro_calls; pub(crate) mod macro_evaluation; pub(crate) mod macro_expansion; pub(crate) mod macro_expansion_facts; +pub(crate) mod macro_expansion_result; pub(crate) mod macro_generated_references; pub(crate) mod macro_modules; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs index 913b13d099..2ea4348538 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::render::{ declaration_expansion, declaration_indexes, macro_arguments, macro_calls, macro_expansion, - macro_modules, + macro_expansion_result, macro_modules, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -13,6 +13,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { macro_arguments::register(module)?; macro_calls::register(module)?; macro_expansion::register(module)?; + macro_expansion_result::register(module)?; macro_modules::register(module)?; Ok(()) } diff --git a/python-boundary.toml b/python-boundary.toml index 375fee0492..4afc3efe7e 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 64690 +remaining_lines = 63824 [[stays]] path = "executor" @@ -189,6 +189,16 @@ path = "compiler/python_nodes/_helpers/identity.py" reason = "user-code" note = "Identity hashing over live user functions (source and globals)." +[[stays]] +path = "compiler/compile/_helpers/render/user_macros.py" +reason = "user-code" +note = "Loads and executes project macro modules, and builds the declaration and enum mappings user macros read." + +[[stays]] +path = "compiler/macro_bridge/classes/macro_bridge.py" +reason = "user-code" +note = "Tracks the loaded user modules macro call results depend on, and runs a render stage through the bridge." + [[stays]] path = "compiler/python_nodes/_helpers/user_inventory.py" reason = "user-code" diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index e8d00f93b9..11779304c8 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -20,7 +20,7 @@ ALLOWED_MACRO_LOAD_PATHS: frozenset[str] = frozenset( { "src/sqlbuild/compiler/compile/main/sql_expansion_context.py", - "src/sqlbuild/compiler/compile/_helpers/render/macros.py", + "src/sqlbuild/compiler/compile/_helpers/render/user_macros.py", } ) ALLOWED_PARAMETER_MUTATION_COMMENT: str = "# sc: allow-param-mutation" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5090770383..d0c7f72284 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -98,6 +98,7 @@ from sqlbuild.compiler.compile.models import ( InferredColumn, LoadedMacro, MacroContext, + MacroExpansionResult, ModelInputBuildContext, NativeModelConfigInputs, NativeModelConfigSession, @@ -2650,7 +2651,18 @@ def substitute_placeholder_defaults(sql: str, placeholders: list[tuple[str, str] def fixture_projection_facts( sql: str, dialect: str | None, / ) -> tuple[list[tuple[str, str]], list[str], list[str]]: ... -def macro_call_names(sql: str, nested: bool, /) -> tuple[list[str] | None, str | None]: ... +def find_macro_call_names(sql: str, /) -> tuple[str, ...]: ... +def expand_sql_macros( + *, + sql: str, + file_path: Path, + loaded_macros: dict[str, LoadedMacro], + macro_context: MacroContext, + macro_overrides: dict[str, str] | None = None, + declaration_resolver: DeclarationScopeResolver | None = None, + declarations: DeclarationResolutionContext | None = None, + consumer: ResourceIdentity | DeclarationIdentity | None = None, +) -> MacroExpansionResult: ... def inventory_macro_modules( sources: list[ tuple[str, str, list[str], list[str] | None, list[str] | None, str, str, str | None] diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py index 35eabc4412..2e3cd8e2e6 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py @@ -6,13 +6,10 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import render_project_var_text +from sqlbuild._native import expand_sql_macros, render_project_var_text from sqlbuild.compiler.compile._helpers.render.declarations import ( expand_scanned_declaration_references, ) -from sqlbuild.compiler.compile._helpers.render.macros import ( - expand_sql_macros_with_spans, -) from sqlbuild.compiler.compile._helpers.render.templating import record_template_reads from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -23,6 +20,7 @@ ExpansionSpan, LoadedMacro, MacroContext, + MacroExpansionResult, SqlInterpolation, ) from sqlbuild.compiler.compile.types import TypedSqlValueRenderer @@ -76,16 +74,14 @@ def expand_authored_sql_with_spans( ) declaration_expanded_sql: str = declaration_result.sql declaration_spans: tuple[ExpansionSpan, ...] = declaration_result.spans - macro_expanded_sql: str - macro_spans: tuple[ExpansionSpan, ...] - macro_expanded_sql, macro_spans = expand_sql_macros_with_spans( + expansion: MacroExpansionResult = expand_sql_macros( sql=declaration_expanded_sql, file_path=file_path, loaded_macros=loaded_macros, macro_context=macro_context, declaration_resolver=declaration_resolver, ) - return macro_expanded_sql, (interpolation_spans, declaration_spans, macro_spans) + return expansion.sql, (interpolation_spans, declaration_spans, expansion.spans) def substitute_sql_vars_with_spans( diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py similarity index 71% rename from src/sqlbuild/compiler/compile/_helpers/render/macros.py rename to src/sqlbuild/compiler/compile/_helpers/render/user_macros.py index 649e7edd91..e48e6316ca 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py @@ -1,4 +1,4 @@ -"""Compile-time expansion helpers for authored project SQL macros.""" +"""Project macro modules loaded and run as user code, and the context user macros read.""" from __future__ import annotations @@ -10,35 +10,20 @@ import threading from collections.abc import Callable, Iterator, Mapping from dataclasses import dataclass, replace -from functools import cache, partial +from functools import partial from pathlib import Path from types import MappingProxyType, ModuleType from typing import cast import sqlbuild._native as _native -from sqlbuild._native import explicit_references_help -from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic -from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( - call_site_sql_references, - typed_reference_kinds, -) -from sqlbuild.compiler.compile.constants import ( - DECLARATION_REFERENCE_NAMES, - MACRO_TOKEN, -) from sqlbuild.compiler.compile.exceptions import ( CompileInputError, - MacroArgumentError, MacroDeclarationLookupError, ) from sqlbuild.compiler.compile.models import ( - CompilerDiagnostic, DeclarationResolutionContext, - DeclarationScopeResolver, - ExpansionSpan, LoadedMacro, MacroContext, - MacroExpansionResult, StaticMacroExport, StaticMacroFault, StaticMacroInventory, @@ -48,24 +33,14 @@ DiscoveredMacroFile, EnumDeclaration, ) -from sqlbuild.compiler.macro_bridge.constants import ( - GENERATED_REFERENCE_MARKERS, -) -from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( validate_resource_identity, ) from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, - ResourceIdentity, - UsageRecord, ) -from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind, UsageKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.python_nodes.models import RELATION_RENDERER, SqlResourceRef -from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.spec.contracts.models import SourceLocation +from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.sql_values.models import SqlValue from sqlbuild.sql_values.types import SqlValueKind @@ -398,157 +373,6 @@ def _rewrite_absolute_project_imports( return ast.Module(body=body, type_ignores=tree.type_ignores) -def expand_sql_macros( - *, - sql: str, - file_path: Path, - loaded_macros: dict[str, LoadedMacro], - macro_overrides: dict[str, str] | None = None, - macro_context: MacroContext, - declaration_resolver: DeclarationScopeResolver | None = None, - consumer: ResourceIdentity | DeclarationIdentity | None = None, -) -> str: - """Expand authored Python macros in one executable SQL string.""" - - expanded_sql: str - expanded_sql, _spans = expand_sql_macros_with_spans( - sql=sql, - file_path=file_path, - loaded_macros=loaded_macros, - macro_overrides=macro_overrides, - macro_context=macro_context, - declaration_resolver=declaration_resolver, - consumer=consumer, - ) - return expanded_sql - - -def expand_sql_macros_with_spans( - *, - sql: str, - file_path: Path, - loaded_macros: dict[str, LoadedMacro], - macro_overrides: dict[str, str] | None = None, - macro_context: MacroContext, - declaration_resolver: DeclarationScopeResolver | None = None, - consumer: ResourceIdentity | DeclarationIdentity | None = None, -) -> tuple[str, tuple[ExpansionSpan, ...]]: - """Expand authored Python macros, returning the span of every substitution.""" - - result: MacroExpansionResult = expand_sql_macros_result( - sql=sql, - file_path=file_path, - loaded_macros=loaded_macros, - macro_overrides=macro_overrides, - macro_context=macro_context, - declaration_resolver=declaration_resolver, - consumer=consumer, - ) - return result.sql, result.spans - - -def expand_sql_macros_result( - *, - sql: str, - file_path: Path, - loaded_macros: dict[str, LoadedMacro], - macro_overrides: dict[str, str] | None = None, - macro_context: MacroContext, - declaration_resolver: DeclarationScopeResolver | None = None, - declarations: DeclarationResolutionContext | None = None, - consumer: ResourceIdentity | DeclarationIdentity | None = None, -) -> MacroExpansionResult: - """Expand macros and retain deterministic resolved dependency facts.""" - - if MACRO_TOKEN not in sql: - return MacroExpansionResult(sql=sql) - if declarations is None and declaration_resolver is not None: - from sqlbuild.compiler.compile._helpers.render.declarations import ( - resolve_declaration_context, - ) - - declarations = resolve_declaration_context( - resolver=declaration_resolver, file_path=file_path - ) - expansion: _native.NativeMacroExpansion = _native.NativeMacroExpansion( - file_path, - ( - loaded_macros, - {} if macro_overrides is None else macro_overrides, - macro_context, - declarations, - consumer, - ), - macro_context._enforce_explicit_references, - macro_expansion_classes(), - ) - expanded_sql, spans, argument_references, dependencies, usages = expansion.expand( - sql, active_macro_bridge() - ) - return MacroExpansionResult( - sql=expanded_sql, - spans=tuple( - ExpansionSpan( - source_start=source_start, - source_end=source_end, - output_start=output_start, - output_end=output_end, - ) - for source_start, source_end, output_start, output_end in spans - ), - argument_references=call_site_sql_references( - tuple( - SqlResourceRef(kind=SqlResourceRefKind(kind), name=name) - for kind, name in argument_references - ) - ), - dependencies=tuple(dict.fromkeys(dependencies)), - usages=tuple(dict.fromkeys(usages)), - ) - - -@cache -def macro_expansion_classes() -> dict[str, object]: - """The classes, enums and helpers the native expansion builds facts and errors from.""" - - from sqlbuild.compiler.frontier.exceptions import NativeStageMismatchError - - return { - "compile_error": CompileInputError, - "macro_argument_error": MacroArgumentError, - "mismatch_error": NativeStageMismatchError, - "declaration_identity": DeclarationIdentity, - "declaration_kind": DeclarationKind, - "macro_kind": DeclarationKind.MACRO, - "usage_record": UsageRecord, - "runtime_usage": UsageKind.RUNTIME, - "resource_identity": ResourceIdentity, - "model_kind": ResourceKind.MODEL, - "resource_ref": SqlResourceRef, - "ref_kinds": typed_reference_kinds(), - "ref_functions": { - kind.value: function for function, kind in typed_reference_kinds().items() - }, - "store_tokens": { - "macro_token": _native.macro_store_token, - "context_token": _native.context_store_token, - }, - "relation_renderer": RELATION_RENDERER, - "invocation_context": macro_invocation_context, - "generated": { - "compiler_diagnostic": CompilerDiagnostic, - "report_diagnostic": report_compile_diagnostic, - "source_location": SourceLocation, - "resource_identity": ResourceIdentity, - "generic_syntax": SqlLexicalSyntax().native_mapping, - "generated_help": explicit_references_help(allowed="macro-generated references"), - "all_markers": list(GENERATED_REFERENCE_MARKERS), - }, - "int_type": int, - "float_type": float, - } - - def _inventory_project_macros( *, macro_files: tuple[DiscoveredMacroFile, ...] ) -> StaticMacroInventory: @@ -562,27 +386,6 @@ def _inventory_project_macros( return inventory -def find_macro_call_names(sql: str) -> tuple[str, ...]: - """Return unique authored macro call names in encounter order.""" - - return tuple( - name - for name in _native_macro_call_names(sql=sql, nested=False) - if name not in DECLARATION_REFERENCE_NAMES - ) - - -def _native_macro_call_names(*, sql: str, nested: bool) -> tuple[str, ...]: - """Every call name Python's scan finds, declaration calls included.""" - - names: list[str] | None - message: str | None - names, message = _native.macro_call_names(sql, nested) - if names is None: - raise CompileInputError(message or "") - return tuple(names) - - def macro_invocation_context( *, expansion: _native.NativeMacroExpansion, diff --git a/src/sqlbuild/compiler/compile/main/_static_macro_inventory.py b/src/sqlbuild/compiler/compile/main/_static_macro_inventory.py index 91c35145a9..b261bc641c 100644 --- a/src/sqlbuild/compiler/compile/main/_static_macro_inventory.py +++ b/src/sqlbuild/compiler/compile/main/_static_macro_inventory.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.compile._helpers.render.macros import _inventory_project_macros +from sqlbuild.compiler.compile._helpers.render.user_macros import _inventory_project_macros from sqlbuild.compiler.compile.models import StaticMacroInventory from sqlbuild.compiler.discovery.models import DiscoveredMacroFile diff --git a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py index d3b65cd047..1978725d64 100644 --- a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py +++ b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py @@ -14,7 +14,7 @@ from sqlbuild.compiler.compile._helpers.render.declarations import ( build_model_declaration_indexes, ) -from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros +from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.models import ( ConstantDeclaration, DeclarationResolutionContext, diff --git a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py index c46880de37..76e2821122 100644 --- a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py +++ b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py @@ -1,9 +1,11 @@ -"""Per-compile state of the native macro bridge: call-site scans, call classes and the memo.""" +"""The native macro bridge over loaded user modules, and the render stage run through it.""" from __future__ import annotations import logging import sys +from collections.abc import Callable +from contextvars import Token from pathlib import Path import sqlbuild._native as _native @@ -17,7 +19,11 @@ project_fingerprint, store_environment, ) -from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME +from sqlbuild.compiler.macro_bridge.constants import ( + ACTIVE_MACRO_BRIDGE, + MACRO_CALL_STORE_FILE_NAME, + STORE_MODULE_PATHS, +) from sqlbuild.compiler.macro_bridge.models import ( ModuleSources, ) @@ -135,3 +141,28 @@ def store_stats(self) -> tuple[int, int]: """Calls found in the store and calls recorded into it.""" return self.memo.store_stats() + + +def run_with_macro_bridge[T](*, stage: Callable[[], T]) -> T: + """Run `stage` once with the bridge; its errors are the Python stage's, raised as they occur.""" + + bridge: MacroBridge = MacroBridge() + token: Token[object | None] = ACTIVE_MACRO_BRIDGE.set(bridge) + try: + result: T = stage() + finally: + ACTIVE_MACRO_BRIDGE.reset(token) + bridge.save_store() + STORE_MODULE_PATHS.update(bridge.store_module_paths()) + hits, misses, recorded = bridge.stats() + store_hits, store_records = bridge.store_stats() + logging.getLogger(__name__).debug( + "Native macro bridge: %d memo hits, %d store hits, %d misses, %d recorded calls, " + "%d stored calls", + hits, + store_hits, + misses, + recorded, + store_records, + ) + return result diff --git a/src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py b/src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py deleted file mode 100644 index ec6ccc31bd..0000000000 --- a/src/sqlbuild/compiler/macro_bridge/main/run_with_macro_bridge.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Run one render stage through the native macro bridge, storing its calls on success.""" - -from __future__ import annotations - -import logging -from collections.abc import Callable -from contextvars import Token - -from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge -from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE, STORE_MODULE_PATHS - - -def run_with_macro_bridge[T](*, stage: Callable[[], T]) -> T: - """Run `stage` once with the bridge; its errors are the Python stage's, raised as they occur.""" - - bridge: MacroBridge = MacroBridge() - token: Token[object | None] = ACTIVE_MACRO_BRIDGE.set(bridge) - try: - result: T = stage() - finally: - ACTIVE_MACRO_BRIDGE.reset(token) - bridge.save_store() - STORE_MODULE_PATHS.update(bridge.store_module_paths()) - hits, misses, recorded = bridge.stats() - store_hits, store_records = bridge.store_stats() - logging.getLogger(__name__).debug( - "Native macro bridge: %d memo hits, %d store hits, %d misses, %d recorded calls, " - "%d stored calls", - hits, - store_hits, - misses, - recorded, - store_records, - ) - return result diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 139135b6d9..87b1d933fb 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -12,7 +12,7 @@ from _pytest.capture import CaptureResult import sqlbuild._native as native_module -import sqlbuild.compiler.compile._helpers.render.macros as macros_module +import sqlbuild.compiler.compile._helpers.render.user_macros as macros_module import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module import sqlbuild.rule_engine._helpers.host.custom_host_pool as custom_host_pool_module import sqlbuild.rule_engine._helpers.run.rule_threads as rule_threads_module diff --git a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py index b174b5e46b..c57f4e8979 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py @@ -8,8 +8,7 @@ from typing import Any import sqlbuild._native as _native -from sqlbuild._native import scan_sql_reference_calls -from sqlbuild.compiler.compile._helpers.render.macros import find_macro_call_names +from sqlbuild._native import find_macro_call_names, scan_sql_reference_calls from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( SqlReferenceScan, diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py index 57c19d9ea2..9607dae950 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -18,11 +18,11 @@ build_compile_inputs, compile_frontier, discover_project_inputs, + expand_sql_macros, mask_capture_noise, render_stage_capture, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile._helpers.render.macros import expand_sql_macros_result from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompileAdapterContext, @@ -343,7 +343,7 @@ def expansion_outcome(*, sql: str, bridge: MacroBridge | None) -> tuple[object, capture: ExpansionFailureCapture = ExpansionFailureCapture() rendered: list[tuple[object, ...]] = [()] with capture: - result: MacroExpansionResult = expand_sql_macros_result( + result: MacroExpansionResult = expand_sql_macros( sql=sql, file_path=_DIFFERENTIAL_FILE, loaded_macros=DIFFERENTIAL_MACROS, diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index 81341445f9..fd191cfe21 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -19,7 +19,7 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_context -from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros +from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( DeclarationResolutionContext, diff --git a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py index 4a091b1aab..1cce92fb83 100644 --- a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py +++ b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py @@ -276,7 +276,7 @@ def test_given_freshness_code_when_checking_sql_ownership_then_matches_contract( path="src/sqlbuild/example/main/build.py", source=dedent( """ - from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros + from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros loaded = load_project_macros(()) """ diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index ea6da6a17f..8bae1a3e59 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -9,6 +9,7 @@ assemble_project, build_compile_inputs, discover_project_inputs, + expand_sql_macros, infer_native_query_columns, project_lineage_views, ) @@ -23,10 +24,7 @@ from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( collect_compile_diagnostics, ) -from sqlbuild.compiler.compile._helpers.render.macros import ( - expand_sql_macros, - load_project_macros, -) +from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompiledDirectLogicSqlTestPayload, @@ -325,7 +323,7 @@ def expand_typed_macro_sql( _enforce_explicit_references=enforce_explicit, ), consumer=ResourceIdentity(kind=ResourceKind.MODEL, name="order_summary"), - ) + ).sql def resolve_model_references_for_adapter(*, sql: str, adapter_name: str) -> str: diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py index 5060952341..17e5d1a51b 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py @@ -5,12 +5,9 @@ import pytest -from sqlbuild.compiler.compile._helpers.render import macros as macros_module -from sqlbuild.compiler.compile._helpers.render.macros import ( - expand_sql_macros, - find_macro_call_names, - load_project_macros, -) +from sqlbuild._native import expand_sql_macros, find_macro_call_names +from sqlbuild.compiler.compile._helpers.render import user_macros as macros_module +from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.models import ( LoadedMacro, MacroContext, @@ -641,7 +638,7 @@ def test_given_sql_macro_variants_when_expanding_then_it_returns_expected_sql( loaded_macros=loaded_macros, macro_overrides=test_case.macro_overrides, macro_context=_MACRO_CONTEXT, - ) + ).sql assert expanded_sql == test_case.expected_sql diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py index 935e5a6369..5fdf06bc65 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py @@ -12,7 +12,7 @@ discover_project_inputs, rebind_declaration_scope, ) -from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros +from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.scopes.models import DeclarationIdentity diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/test_run_with_macro_bridge.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/test_run_with_macro_bridge.py index cb996b6853..bae714054e 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/test_run_with_macro_bridge.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/test_run_with_macro_bridge.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import run_with_macro_bridge from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge -from sqlbuild.compiler.macro_bridge.main.run_with_macro_bridge import run_with_macro_bridge from tests.unit.src.sqlbuild.compiler.macro_bridge.main._test_types import ( MacroBridgeFailureTestCase, MacroBridgeSuccessTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index b559748ef1..8106ea1ef6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -8,10 +8,10 @@ from types import MappingProxyType from typing import Any +from sqlbuild._native import expand_sql_macros from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile._helpers.render.macros import expand_sql_macros from sqlbuild.compiler.compile.constants import ( ASSERT_TEST_CTE_PREFIX, DBT_REF_TEST_CTE_PREFIX, @@ -427,7 +427,7 @@ def _expand_model_query_overrides( loaded_macros=loaded_macros, macro_overrides=test_case.macro_mocks, macro_context=_MACRO_CONTEXT, - ) + ).sql def _keep_model_query_overrides( From 0818b844f513b9f03ac77e5cfd99bb6eb37ccbc0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:41:24 +0100 Subject: [PATCH 19/24] chore: keep the macro store environment fingerprint as user-code --- python-boundary.toml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 4afc3efe7e..16ad19959b 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 63824 +remaining_lines = 63635 [[stays]] path = "executor" @@ -189,6 +189,11 @@ path = "compiler/python_nodes/_helpers/identity.py" reason = "user-code" note = "Identity hashing over live user functions (source and globals)." +[[stays]] +path = "compiler/macro_bridge/_helpers/store_environment.py" +reason = "user-code" +note = "Fingerprints the Python interpreter and installed packages that project macros import from; the macro call store cache key." + [[stays]] path = "compiler/compile/_helpers/render/user_macros.py" reason = "user-code" From 6f43892be26697db4a90e1722a52fcd0ed44aee5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 01:55:44 +0100 Subject: [PATCH 20/24] refactor: build the combined dbt graph and resolve sqb dbt selection natively --- .../src/dbt_graph/_helpers/closure.rs | 50 +++ .../src/dbt_graph/_helpers/edges.rs | 47 +++ .../src/dbt_graph/_helpers/endpoints.rs | 57 +++ .../src/dbt_graph/_helpers/mod.rs | 4 + .../src/dbt_graph/_helpers/selection.rs | 148 ++++++++ .../sqlbuild-core/src/dbt_graph/constants.rs | 16 + crates/sqlbuild-core/src/dbt_graph/errors.rs | 8 + .../main/build_dbt_combined_graph.rs | 71 ++++ .../main/dbt_downstream_sqlbuild_models.rs | 27 ++ .../dbt_graph/main/dbt_fqn_selector_term.rs | 12 + .../sqlbuild-core/src/dbt_graph/main/mod.rs | 4 + .../dbt_graph/main/resolve_dbt_selection.rs | 21 ++ crates/sqlbuild-core/src/dbt_graph/mod.rs | 11 + crates/sqlbuild-core/src/dbt_graph/models.rs | 149 ++++++++ crates/sqlbuild-core/src/dbt_graph/tests.rs | 6 + .../src/dbt_graph/tests/helpers.rs | 144 ++++++++ .../src/dbt_graph/tests/test_dbt_graph.rs | 202 +++++++++++ .../src/dbt_graph/tests/test_types.rs | 26 ++ crates/sqlbuild-core/src/dbt_graph/types.rs | 13 + crates/sqlbuild-core/src/lib.rs | 1 + .../bindings/_helpers/dbt_interop/graph.rs | 343 ++++++++++++++++++ .../bindings/_helpers/dbt_interop/manifest.rs | 2 +- .../src/bindings/_helpers/dbt_interop/mod.rs | 1 + .../_helpers/dbt_interop/registration.rs | 3 +- src/sqlbuild/_native.pyi | 23 ++ .../integrations/dbt/_helpers/graph/core.py | 163 --------- .../dbt/_helpers/pipeline/execute.py | 29 +- .../dbt/_helpers/pipeline/interop_prologue.py | 4 +- .../dbt/_helpers/planning/orchestration.py | 11 +- .../dbt/_helpers/selection/core.py | 248 ------------- .../dbt/_helpers/selection/selector_terms.py | 11 - .../integrations/dbt/main/graph/__init__.py | 1 - .../dbt/main/graph/_build_combined_graph.py | 13 - .../dbt/main/graph/_dbt_model_graph_key.py | 10 - .../main/graph/_sqlbuild_model_graph_key.py | 10 - .../main/planning/_plan_interop_command.py | 2 +- .../dbt/main/selection/__init__.py | 1 - .../main/selection/_dbt_fqn_selector_term.py | 11 - src/sqlbuild/integrations/dbt/models.py | 46 +-- src/sqlbuild/integrations/dbt/types.py | 14 - .../sqlbuild/integrations/dbt/test_graph.py | 17 +- .../src/sqlbuild/integrations/dbt/helpers.py | 75 ++-- .../sqlbuild/integrations/dbt/test_execute.py | 55 +-- .../sqlbuild/integrations/dbt/test_graph.py | 30 +- .../dbt/test_plan_orchestration.py | 4 +- .../integrations/dbt/test_selection.py | 15 +- 46 files changed, 1463 insertions(+), 696 deletions(-) create mode 100644 crates/sqlbuild-core/src/dbt_graph/_helpers/closure.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/_helpers/edges.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/_helpers/endpoints.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/_helpers/selection.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/constants.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/errors.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/main/build_dbt_combined_graph.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/main/dbt_downstream_sqlbuild_models.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/main/dbt_fqn_selector_term.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/main/mod.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/main/resolve_dbt_selection.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/mod.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/models.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/tests.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/tests/test_dbt_graph.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/tests/test_types.rs create mode 100644 crates/sqlbuild-core/src/dbt_graph/types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs delete mode 100644 src/sqlbuild/integrations/dbt/_helpers/graph/core.py delete mode 100644 src/sqlbuild/integrations/dbt/_helpers/selection/core.py delete mode 100644 src/sqlbuild/integrations/dbt/_helpers/selection/selector_terms.py delete mode 100644 src/sqlbuild/integrations/dbt/main/graph/__init__.py delete mode 100644 src/sqlbuild/integrations/dbt/main/graph/_build_combined_graph.py delete mode 100644 src/sqlbuild/integrations/dbt/main/graph/_dbt_model_graph_key.py delete mode 100644 src/sqlbuild/integrations/dbt/main/graph/_sqlbuild_model_graph_key.py delete mode 100644 src/sqlbuild/integrations/dbt/main/selection/__init__.py delete mode 100644 src/sqlbuild/integrations/dbt/main/selection/_dbt_fqn_selector_term.py diff --git a/crates/sqlbuild-core/src/dbt_graph/_helpers/closure.rs b/crates/sqlbuild-core/src/dbt_graph/_helpers/closure.rs new file mode 100644 index 0000000000..f51d0a2dc1 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/_helpers/closure.rs @@ -0,0 +1,50 @@ +//! Breadth-first reachability and path nodes over combined graph edges. + +use std::collections::{HashSet, VecDeque}; + +use crate::dbt_graph::models::{DbtGraphEdges, DbtGraphKey}; + +/// Every key reachable from `start`; `start` itself only through a cycle. +pub(crate) fn reachable(start: &DbtGraphKey, edges: &DbtGraphEdges) -> HashSet { + let mut visited: HashSet = HashSet::new(); + let mut frontier: VecDeque<&DbtGraphKey> = VecDeque::from([start]); + while let Some(current) = frontier.pop_front() { + for neighbor in edges.targets.get(current).into_iter().flatten() { + if visited.insert(neighbor.clone()) { + frontier.push_back(neighbor); + } + } + } + visited +} + +/// Keys on directed paths from `start` to `end`, or `None` when `end` is not downstream. +pub(crate) fn path_nodes( + (start, end): (&DbtGraphKey, &DbtGraphKey), + downstream: &DbtGraphEdges, + upstream: &DbtGraphEdges, +) -> Option> { + let reachable_keys: HashSet = reachable(start, downstream); + if !reachable_keys.contains(end) { + return None; + } + let mut upstream_from_end: HashSet = HashSet::new(); + let mut stack: Vec<&DbtGraphKey> = vec![end]; + while let Some(current) = stack.pop() { + if !upstream_from_end.insert(current.clone()) { + continue; + } + for parent in upstream.targets.get(current).into_iter().flatten() { + if reachable_keys.contains(parent) || parent == start { + stack.push(parent); + } + } + } + let mut nodes: HashSet = reachable_keys + .intersection(&upstream_from_end) + .cloned() + .collect(); + nodes.insert(start.clone()); + nodes.insert(end.clone()); + Some(nodes) +} diff --git a/crates/sqlbuild-core/src/dbt_graph/_helpers/edges.rs b/crates/sqlbuild-core/src/dbt_graph/_helpers/edges.rs new file mode 100644 index 0000000000..942628f39e --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/_helpers/edges.rs @@ -0,0 +1,47 @@ +//! Insertion-ordered edge maps: adding nodes and edges, sorting and inversion. + +use crate::dbt_graph::models::{DbtGraphEdges, DbtGraphKey}; + +/// `edges` with `key` added with no targets unless it is already present. +pub(crate) fn ensure(mut edges: DbtGraphEdges, key: &DbtGraphKey) -> DbtGraphEdges { + if !edges.targets.contains_key(key) { + edges.order.push(key.clone()); + edges.targets.insert(key.clone(), Vec::new()); + } + edges +} + +/// `edges` with one edge appended from `source`, adding `source` first when it is new. +pub(crate) fn push( + edges: DbtGraphEdges, + source: &DbtGraphKey, + target: DbtGraphKey, +) -> DbtGraphEdges { + let mut edges: DbtGraphEdges = ensure(edges, source); + if let Some(targets) = edges.targets.get_mut(source) { + targets.push(target); + } + edges +} + +/// `edges` with every target list sorted by stable ID, duplicates kept. +pub(crate) fn sort_targets(mut edges: DbtGraphEdges) -> DbtGraphEdges { + for targets in edges.targets.values_mut() { + targets.sort_by_cached_key(DbtGraphKey::stable_id); + } + edges +} + +/// Reversed edges: every source first, then each new target in first-seen order. +pub(crate) fn inverted(edges: &DbtGraphEdges) -> DbtGraphEdges { + let mut result: DbtGraphEdges = DbtGraphEdges::default(); + for key in &edges.order { + result = ensure(result, key); + } + for source in &edges.order { + for target in edges.targets.get(source).into_iter().flatten() { + result = push(result, target, source.clone()); + } + } + result +} diff --git a/crates/sqlbuild-core/src/dbt_graph/_helpers/endpoints.rs b/crates/sqlbuild-core/src/dbt_graph/_helpers/endpoints.rs new file mode 100644 index 0000000000..08ae9d6c7a --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/_helpers/endpoints.rs @@ -0,0 +1,57 @@ +//! Path selector endpoints: splitting a `start~end` core and resolving each name. + +use crate::dbt_graph::constants::PATH_SELECTOR_SEPARATOR; +use crate::dbt_graph::errors::DbtSelectionError; +use crate::dbt_graph::models::DbtGraphKey; +use crate::dbt_graph::types::DbtSelectorResolver; +use crate::text::main::python_strip::python_strip; + +/// The stripped endpoint names around the first separator of a path core. +pub(crate) fn endpoint_names<'a, E>( + core: &'a str, + raw_term: &str, +) -> Result<(&'a str, &'a str), DbtSelectionError> { + let (start, end) = core + .split_once(PATH_SELECTOR_SEPARATOR) + .unwrap_or((core, "")); + let (start, end) = (python_strip(start), python_strip(end)); + if start.is_empty() || end.is_empty() { + return Err(DbtSelectionError::PathSelector(format!( + "path selector '{raw_term}' requires names on both sides of '~'" + ))); + } + Ok((start, end)) +} + +/// The graph key one endpoint names: a SQLBuild model first, else a unique dbt model. +pub(crate) fn endpoint_key( + resolver: &impl DbtSelectorResolver, + manifest_models: Option<&[(String, String)]>, + (endpoint, raw_term): (&str, &str), +) -> Result> { + let (names, _translation) = resolver + .model_names(endpoint) + .map_err(DbtSelectionError::Resolver)?; + if let Some(name) = names.first() { + return Ok(DbtGraphKey::sqlbuild_model(name)); + } + let Some(manifest_models) = manifest_models else { + return Err(DbtSelectionError::PathSelector(format!( + "path selector '{raw_term}' endpoint '{endpoint}' is not a SQLBuild model" + ))); + }; + let matches: Vec<&String> = manifest_models + .iter() + .filter(|(_, name)| name == endpoint) + .map(|(unique_id, _)| unique_id) + .collect(); + match matches.as_slice() { + [unique_id] => Ok(DbtGraphKey::dbt_model(unique_id)), + [] => Err(DbtSelectionError::PathSelector(format!( + "path selector '{raw_term}' endpoint '{endpoint}' is not a known SQLBuild or dbt model" + ))), + _ => Err(DbtSelectionError::PathSelector(format!( + "path selector '{raw_term}' endpoint '{endpoint}' is ambiguous across dbt models" + ))), + } +} diff --git a/crates/sqlbuild-core/src/dbt_graph/_helpers/mod.rs b/crates/sqlbuild-core/src/dbt_graph/_helpers/mod.rs new file mode 100644 index 0000000000..ced417d413 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod closure; +pub(crate) mod edges; +pub(crate) mod endpoints; +pub(crate) mod selection; diff --git a/crates/sqlbuild-core/src/dbt_graph/_helpers/selection.rs b/crates/sqlbuild-core/src/dbt_graph/_helpers/selection.rs new file mode 100644 index 0000000000..a4a4703bab --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/_helpers/selection.rs @@ -0,0 +1,148 @@ +//! One selector term at a time: path selectors, SQLBuild selectors and dbt anchors. + +use std::collections::HashSet; + +use crate::dbt_graph::_helpers::closure::{path_nodes, reachable}; +use crate::dbt_graph::_helpers::endpoints::{endpoint_key, endpoint_names}; +use crate::dbt_graph::constants::PATH_SELECTOR_SEPARATOR; +use crate::dbt_graph::errors::DbtSelectionError; +use crate::dbt_graph::models::{ + DbtCombinedGraph, DbtGraphKey, DbtGraphOwner, DbtSelection, DbtSelectionRequest, + DbtSelectionState, DbtSelectorExpansion, +}; +use crate::dbt_graph::types::DbtSelectorResolver; + +fn added( + mut state: DbtSelectionState, + keys: impl IntoIterator, +) -> DbtSelectionState { + for key in keys { + match key.owner { + DbtGraphOwner::Sqlbuild => state.sqlbuild.insert(key.name), + DbtGraphOwner::Dbt => state.dbt.insert(key.name), + }; + } + state +} + +fn expanded( + keys: HashSet, + parsed: &DbtSelectorExpansion, + graph: &DbtCombinedGraph, +) -> HashSet { + let mut result: HashSet = keys.clone(); + if parsed.upstream { + for key in &keys { + result.extend(reachable(key, &graph.upstream)); + } + } + if parsed.downstream { + for key in &keys { + result.extend(reachable(key, &graph.downstream)); + } + } + result +} + +fn path_keys( + resolver: &impl DbtSelectorResolver, + (graph, request): (&DbtCombinedGraph, &DbtSelectionRequest<'_>), + (core, raw_term): (&str, &str), +) -> Result, DbtSelectionError> { + let (start_name, end_name) = endpoint_names(core, raw_term)?; + let start: DbtGraphKey = + endpoint_key(resolver, request.manifest_models, (start_name, raw_term))?; + let end: DbtGraphKey = endpoint_key(resolver, request.manifest_models, (end_name, raw_term))?; + Ok(path_nodes((&start, &end), &graph.downstream, &graph.upstream).unwrap_or_default()) +} + +fn with_anchor(mut selection: DbtSelection, term: &str, unique_ids: &[String]) -> DbtSelection { + selection.dbt_anchor_terms.push(term.to_owned()); + match selection + .dbt_anchor_unique_ids_by_term + .iter_mut() + .find(|(existing, _)| existing == term) + { + Some((_, existing)) => unique_ids.clone_into(existing), + None => selection + .dbt_anchor_unique_ids_by_term + .push((term.to_owned(), unique_ids.to_vec())), + } + selection +} + +fn anchored( + mut state: DbtSelectionState, + (graph, request): (&DbtCombinedGraph, &DbtSelectionRequest<'_>), + term: &str, +) -> DbtSelectionState { + let anchors: &[String] = request + .anchor_unique_ids_by_term + .get(term) + .map_or(&[], Vec::as_slice); + state.selection = with_anchor(state.selection, term, anchors); + for unique_id in anchors { + state = added( + state, + reachable(&DbtGraphKey::dbt_model(unique_id), &graph.downstream), + ); + } + state +} + +/// `state` with one select term resolved into it. +pub(crate) fn select_term( + resolver: &impl DbtSelectorResolver, + inputs: (&DbtCombinedGraph, &DbtSelectionRequest<'_>), + term: &str, + state: DbtSelectionState, +) -> Result> { + let graph: &DbtCombinedGraph = inputs.0; + let parsed: DbtSelectorExpansion = resolver.split(term).map_err(DbtSelectionError::Resolver)?; + if parsed.core.contains(PATH_SELECTOR_SEPARATOR) { + let keys: HashSet = path_keys(resolver, inputs, (&parsed.core, term))?; + return Ok(added(state, expanded(keys, &parsed, graph))); + } + let (names, translated_path) = resolver + .model_names(&parsed.core) + .map_err(DbtSelectionError::Resolver)?; + let mut state: DbtSelectionState = state; + if let Some(translated_path) = translated_path { + state + .selection + .path_translations + .push((term.to_owned(), translated_path)); + } + if !names.is_empty() { + let keys: HashSet = names + .iter() + .map(|name| DbtGraphKey::sqlbuild_model(name)) + .collect(); + return Ok(added(state, expanded(keys, &parsed, graph))); + } + if !parsed.downstream { + return Ok(state); + } + Ok(anchored(state, inputs, term)) +} + +/// The final selection: sorted names without excluded SQLBuild models. +pub(crate) fn finished( + resolver: &impl DbtSelectorResolver, + exclude: &[String], + state: DbtSelectionState, +) -> Result { + let mut excluded: HashSet = HashSet::new(); + for term in exclude { + let parsed: DbtSelectorExpansion = resolver.split(term)?; + excluded.extend(resolver.model_names(&parsed.core)?.0); + } + let mut selection: DbtSelection = state.selection; + selection.sqlbuild_model_names = state + .sqlbuild + .into_iter() + .filter(|name| !excluded.contains(name)) + .collect(); + selection.dbt_required_unique_ids = state.dbt.into_iter().collect(); + Ok(selection) +} diff --git a/crates/sqlbuild-core/src/dbt_graph/constants.rs b/crates/sqlbuild-core/src/dbt_graph/constants.rs new file mode 100644 index 0000000000..04f7c003d8 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/constants.rs @@ -0,0 +1,16 @@ +//! Graph key namespaces and path selector syntax. + +/// Owner namespace of dbt nodes. +pub const DBT_OWNER: &str = "dbt"; +/// Owner namespace of SQLBuild nodes. +pub const SQLBUILD_OWNER: &str = "sqb"; +/// Resource type namespace of models. +pub const MODEL_RESOURCE_TYPE: &str = "model"; +/// Resource type namespace of dbt sources and seeds. +pub const SOURCE_RESOURCE_TYPE: &str = "source"; +/// Separator between the two endpoints of a path selector. +pub const PATH_SELECTOR_SEPARATOR: &str = "~"; +/// Diagnostic code of an invalid path selector. +pub const PATH_SELECTOR_ERROR_CODE: &str = "C237"; +/// Prefix of an exact dbt FQN selector. +pub const FQN_SELECTOR_PREFIX: &str = "fqn:"; diff --git a/crates/sqlbuild-core/src/dbt_graph/errors.rs b/crates/sqlbuild-core/src/dbt_graph/errors.rs new file mode 100644 index 0000000000..d1d3fdbcca --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/errors.rs @@ -0,0 +1,8 @@ +//! Selector resolution failures. + +/// A resolver failure, or an invalid path selector with its message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DbtSelectionError { + Resolver(E), + PathSelector(String), +} diff --git a/crates/sqlbuild-core/src/dbt_graph/main/build_dbt_combined_graph.rs b/crates/sqlbuild-core/src/dbt_graph/main/build_dbt_combined_graph.rs new file mode 100644 index 0000000000..11518cde1a --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/main/build_dbt_combined_graph.rs @@ -0,0 +1,71 @@ +//! Build the combined graph from manifest models and compiled SQLBuild models. + +use std::collections::HashSet; + +use crate::dbt_graph::_helpers::edges::{ensure, inverted, push, sort_targets}; +use crate::dbt_graph::models::{ + DbtCombinedGraph, DbtGraphEdges, DbtGraphKey, DbtGraphManifest, DbtGraphProjectModel, +}; + +fn add_dbt_edges(mut upstream: DbtGraphEdges, manifest: &DbtGraphManifest) -> DbtGraphEdges { + let model_ids: HashSet<&str> = manifest + .models + .iter() + .map(|model| model.unique_id.as_str()) + .collect(); + for model in &manifest.models { + let key: DbtGraphKey = DbtGraphKey::dbt_model(&model.unique_id); + upstream = ensure(upstream, &key); + for dependency in &model.depends_on_nodes { + if model_ids.contains(dependency.as_str()) { + upstream = push(upstream, &key, DbtGraphKey::dbt_model(dependency)); + } + if manifest.source_ids.contains(dependency) { + let source: DbtGraphKey = DbtGraphKey::dbt_source(dependency); + upstream = push(ensure(upstream, &source), &key, source); + } + if manifest.seed_ids.contains(dependency) { + let seed: DbtGraphKey = DbtGraphKey::dbt_source(dependency); + upstream = push(ensure(upstream, &seed), &key, seed); + } + } + } + upstream +} + +fn add_sqlbuild_edges( + mut upstream: DbtGraphEdges, + models: &[DbtGraphProjectModel], +) -> DbtGraphEdges { + let names: HashSet<&str> = models.iter().map(|model| model.name.as_str()).collect(); + for model in models { + let key: DbtGraphKey = DbtGraphKey::sqlbuild_model(&model.name); + upstream = ensure(upstream, &key); + for dependency in &model.model_deps { + if names.contains(dependency.as_str()) { + upstream = push(upstream, &key, DbtGraphKey::sqlbuild_model(dependency)); + } + } + for unique_id in &model.dbt_ref_unique_ids { + upstream = push(upstream, &key, DbtGraphKey::dbt_model(unique_id)); + } + } + upstream +} + +/// Upstream edges from dbt and SQLBuild dependencies, and their inversion, sorted by stable ID. +#[must_use] +pub fn build_dbt_combined_graph( + manifest: &DbtGraphManifest, + models: &[DbtGraphProjectModel], +) -> DbtCombinedGraph { + let upstream: DbtGraphEdges = sort_targets(add_sqlbuild_edges( + add_dbt_edges(DbtGraphEdges::default(), manifest), + models, + )); + let downstream: DbtGraphEdges = sort_targets(inverted(&upstream)); + DbtCombinedGraph { + upstream, + downstream, + } +} diff --git a/crates/sqlbuild-core/src/dbt_graph/main/dbt_downstream_sqlbuild_models.rs b/crates/sqlbuild-core/src/dbt_graph/main/dbt_downstream_sqlbuild_models.rs new file mode 100644 index 0000000000..d9157a53c7 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/main/dbt_downstream_sqlbuild_models.rs @@ -0,0 +1,27 @@ +//! SQLBuild models downstream of dbt models. + +use std::collections::BTreeSet; + +use crate::dbt_graph::_helpers::closure::reachable; +use crate::dbt_graph::models::{ + DbtCombinedGraph, DbtGraphKey, DbtGraphOwner, DbtGraphResourceType, +}; + +/// Sorted names of SQLBuild models transitively downstream of any of the dbt models. +#[must_use] +pub fn dbt_downstream_sqlbuild_models( + graph: &DbtCombinedGraph, + unique_ids: &[String], +) -> Vec { + let mut names: BTreeSet = BTreeSet::new(); + for unique_id in unique_ids { + for key in reachable(&DbtGraphKey::dbt_model(unique_id), &graph.downstream) { + if key.owner == DbtGraphOwner::Sqlbuild + && key.resource_type == DbtGraphResourceType::Model + { + names.insert(key.name); + } + } + } + names.into_iter().collect() +} diff --git a/crates/sqlbuild-core/src/dbt_graph/main/dbt_fqn_selector_term.rs b/crates/sqlbuild-core/src/dbt_graph/main/dbt_fqn_selector_term.rs new file mode 100644 index 0000000000..028808be43 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/main/dbt_fqn_selector_term.rs @@ -0,0 +1,12 @@ +//! The exact dbt FQN selector of a node. + +use crate::dbt_graph::constants::FQN_SELECTOR_PREFIX; + +/// `fqn:` when the node has an FQN, else `fallback`. +#[must_use] +pub fn dbt_fqn_selector_term(fqn: &[String], fallback: &str) -> String { + if fqn.is_empty() { + return fallback.to_owned(); + } + format!("{FQN_SELECTOR_PREFIX}{}", fqn.join(".")) +} diff --git a/crates/sqlbuild-core/src/dbt_graph/main/mod.rs b/crates/sqlbuild-core/src/dbt_graph/main/mod.rs new file mode 100644 index 0000000000..e72b33618a --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/main/mod.rs @@ -0,0 +1,4 @@ +pub mod build_dbt_combined_graph; +pub mod dbt_downstream_sqlbuild_models; +pub mod dbt_fqn_selector_term; +pub mod resolve_dbt_selection; diff --git a/crates/sqlbuild-core/src/dbt_graph/main/resolve_dbt_selection.rs b/crates/sqlbuild-core/src/dbt_graph/main/resolve_dbt_selection.rs new file mode 100644 index 0000000000..1dec1e9f20 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/main/resolve_dbt_selection.rs @@ -0,0 +1,21 @@ +//! Resolve `sqb dbt` selectors to SQLBuild models and the dbt nodes they require. + +use crate::dbt_graph::_helpers::selection::{finished, select_term}; +use crate::dbt_graph::errors::DbtSelectionError; +use crate::dbt_graph::models::{ + DbtCombinedGraph, DbtSelection, DbtSelectionRequest, DbtSelectionState, +}; +use crate::dbt_graph::types::DbtSelectorResolver; + +/// Resolve each term in order: path selectors, SQLBuild selectors, then trailing-plus dbt anchors. +pub fn resolve_dbt_selection( + resolver: &impl DbtSelectorResolver, + graph: &DbtCombinedGraph, + request: &DbtSelectionRequest<'_>, +) -> Result> { + let mut state: DbtSelectionState = DbtSelectionState::default(); + for term in request.select { + state = select_term(resolver, (graph, request), term, state)?; + } + finished(resolver, request.exclude, state).map_err(DbtSelectionError::Resolver) +} diff --git a/crates/sqlbuild-core/src/dbt_graph/mod.rs b/crates/sqlbuild-core/src/dbt_graph/mod.rs new file mode 100644 index 0000000000..151f4acff0 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/mod.rs @@ -0,0 +1,11 @@ +//! The combined dbt and SQLBuild model graph and `sqb dbt` selector resolution over it. + +mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/dbt_graph/models.rs b/crates/sqlbuild-core/src/dbt_graph/models.rs new file mode 100644 index 0000000000..f851e3dede --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/models.rs @@ -0,0 +1,149 @@ +//! Combined graph keys, inputs and selection results. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use crate::dbt_graph::constants::{ + DBT_OWNER, MODEL_RESOURCE_TYPE, SOURCE_RESOURCE_TYPE, SQLBUILD_OWNER, +}; + +/// The namespace that owns a graph node. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum DbtGraphOwner { + Dbt, + Sqlbuild, +} + +/// The resource type namespace of a graph node. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum DbtGraphResourceType { + Model, + Source, +} + +/// One owner-qualified combined graph node. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct DbtGraphKey { + pub owner: DbtGraphOwner, + pub resource_type: DbtGraphResourceType, + pub name: String, +} + +impl DbtGraphKey { + /// The key of one dbt model. + #[must_use] + pub fn dbt_model(unique_id: &str) -> Self { + Self { + owner: DbtGraphOwner::Dbt, + resource_type: DbtGraphResourceType::Model, + name: unique_id.to_owned(), + } + } + + /// The key of one dbt source or seed. + #[must_use] + pub fn dbt_source(unique_id: &str) -> Self { + Self { + owner: DbtGraphOwner::Dbt, + resource_type: DbtGraphResourceType::Source, + name: unique_id.to_owned(), + } + } + + /// The key of one SQLBuild model. + #[must_use] + pub fn sqlbuild_model(name: &str) -> Self { + Self { + owner: DbtGraphOwner::Sqlbuild, + resource_type: DbtGraphResourceType::Model, + name: name.to_owned(), + } + } + + /// `owner:resource_type:name`, the order graph edges are sorted by. + #[must_use] + pub fn stable_id(&self) -> String { + let owner: &str = match self.owner { + DbtGraphOwner::Dbt => DBT_OWNER, + DbtGraphOwner::Sqlbuild => SQLBUILD_OWNER, + }; + let resource_type: &str = match self.resource_type { + DbtGraphResourceType::Model => MODEL_RESOURCE_TYPE, + DbtGraphResourceType::Source => SOURCE_RESOURCE_TYPE, + }; + format!("{owner}:{resource_type}:{}", self.name) + } +} + +/// Directed edges keyed by node, in first-insertion node order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtGraphEdges { + pub order: Vec, + pub targets: HashMap>, +} + +/// The combined graph with upstream and downstream edges sorted by stable ID. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtCombinedGraph { + pub upstream: DbtGraphEdges, + pub downstream: DbtGraphEdges, +} + +/// One dbt model of the manifest index. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtGraphManifestModel { + pub unique_id: String, + pub name: String, + pub depends_on_nodes: Vec, +} + +/// The manifest nodes the graph reads, models in index order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtGraphManifest { + pub models: Vec, + pub source_ids: HashSet, + pub seed_ids: HashSet, +} + +/// One compiled SQLBuild model: its SQLBuild model dependencies and resolved dbt refs. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtGraphProjectModel { + pub name: String, + pub model_deps: Vec, + pub dbt_ref_unique_ids: Vec, +} + +/// One selector term split into its core and expansion markers. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtSelectorExpansion { + pub core: String, + pub upstream: bool, + pub downstream: bool, +} + +/// The selector terms and dbt anchors one `sqb dbt` selection resolves. +#[derive(Clone, Copy, Debug)] +pub struct DbtSelectionRequest<'a> { + pub select: &'a [String], + pub exclude: &'a [String], + pub anchor_unique_ids_by_term: &'a HashMap>, + /// Manifest models as `(unique_id, name)` in index order, when a manifest is loaded. + pub manifest_models: Option<&'a [(String, String)]>, +} + +/// Selected SQLBuild models and the dbt nodes they require. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtSelection { + pub sqlbuild_model_names: Vec, + pub dbt_required_unique_ids: Vec, + pub dbt_anchor_terms: Vec, + pub dbt_anchor_unique_ids_by_term: Vec<(String, Vec)>, + pub path_translations: Vec<(String, String)>, +} + +/// Selection accumulated over the terms resolved so far. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtSelectionState { + pub sqlbuild: BTreeSet, + pub dbt: BTreeSet, + pub selection: DbtSelection, +} diff --git a/crates/sqlbuild-core/src/dbt_graph/tests.rs b/crates/sqlbuild-core/src/dbt_graph/tests.rs new file mode 100644 index 0000000000..12ca9568aa --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_dbt_graph.rs"] +mod dbt_graph; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/dbt_graph/tests/helpers.rs b/crates/sqlbuild-core/src/dbt_graph/tests/helpers.rs new file mode 100644 index 0000000000..91b2ff9b31 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/tests/helpers.rs @@ -0,0 +1,144 @@ +use std::collections::{HashMap, HashSet}; + +use crate::dbt_graph::errors::DbtSelectionError; +use crate::dbt_graph::main::build_dbt_combined_graph::build_dbt_combined_graph; +use crate::dbt_graph::main::resolve_dbt_selection::resolve_dbt_selection; +use crate::dbt_graph::models::{ + DbtCombinedGraph, DbtGraphEdges, DbtGraphManifest, DbtGraphManifestModel, DbtGraphProjectModel, + DbtSelection, DbtSelectionRequest, DbtSelectorExpansion, +}; +use crate::dbt_graph::tests::test_types::NameResolver; +use crate::dbt_graph::types::DbtSelectorResolver; + +fn texts(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// Manifest: stg <- raw source and seed; int <- stg; a second `dup` name in two packages. +pub(super) fn manifest() -> DbtGraphManifest { + let model = |unique_id: &str, name: &str, deps: &[&str]| DbtGraphManifestModel { + unique_id: unique_id.to_owned(), + name: name.to_owned(), + depends_on_nodes: texts(deps), + }; + DbtGraphManifest { + models: vec![ + model( + "model.a.stg", + "stg", + &["source.a.raw.orders", "seed.a.codes", "x"], + ), + model("model.a.int", "int", &["model.a.stg", "model.a.stg"]), + model("model.a.dup", "dup", &[]), + model("model.b.dup", "dup", &[]), + ], + source_ids: HashSet::from(["source.a.raw.orders".to_owned()]), + seed_ids: HashSet::from(["seed.a.codes".to_owned()]), + } +} + +/// Project: fact <- dbt int; mart <- fact and an unknown model; local alone. +pub(super) fn project() -> Vec { + let model = |name: &str, deps: &[&str], refs: &[&str]| DbtGraphProjectModel { + name: name.to_owned(), + model_deps: texts(deps), + dbt_ref_unique_ids: texts(refs), + }; + vec![ + model("fact", &[], &["model.a.int"]), + model("mart", &["fact", "missing"], &[]), + model("local", &[], &[]), + ] +} + +pub(super) fn graph() -> DbtCombinedGraph { + build_dbt_combined_graph(&manifest(), &project()) +} + +/// Edges as stable IDs in node order. +pub(super) fn stable_edges(edges: &DbtGraphEdges) -> Vec<(String, Vec)> { + edges + .order + .iter() + .map(|key| { + let targets: Vec = edges + .targets + .get(key) + .into_iter() + .flatten() + .map(|target| target.stable_id()) + .collect(); + (key.stable_id(), targets) + }) + .collect() +} + +impl DbtSelectorResolver for NameResolver { + type Error = String; + + fn split(&self, term: &str) -> Result { + let core: &str = term.trim_matches('+'); + if core.is_empty() { + return Err(format!("selector '{term}' has no name")); + } + Ok(DbtSelectorExpansion { + core: core.to_owned(), + upstream: term.starts_with('+'), + downstream: term.ends_with('+'), + }) + } + + fn model_names(&self, core: &str) -> Result<(Vec, Option), String> { + if let Some(path) = core.strip_prefix("path:") { + let translated: String = path.replace('\\', "/"); + let changed: Option = + (translated != path).then(|| format!("path:{translated}")); + return Ok((Vec::new(), changed)); + } + Ok(( + self.names + .iter() + .filter(|name| **name == core) + .map(|name| (*name).to_owned()) + .collect(), + None, + )) + } +} + +/// Resolve selectors against the helper graph with a name-only resolver. +pub(super) fn select( + select: &[&str], + exclude: &[&str], + anchors: &[(&str, Vec<&str>)], +) -> Result> { + let anchors: HashMap> = anchors + .iter() + .map(|(term, ids)| ((*term).to_owned(), texts(ids))) + .collect(); + let manifest_models: Vec<(String, String)> = manifest() + .models + .into_iter() + .map(|model| (model.unique_id, model.name)) + .collect(); + let (select, exclude): (Vec, Vec) = (texts(select), texts(exclude)); + let request: DbtSelectionRequest<'_> = DbtSelectionRequest { + select: &select, + exclude: &exclude, + anchor_unique_ids_by_term: &anchors, + manifest_models: Some(&manifest_models), + }; + let resolver: NameResolver = NameResolver { + names: vec!["fact", "mart", "local"], + }; + resolve_dbt_selection(&resolver, &graph(), &request) +} + +/// A selection with the given sorted models and dbt nodes. +pub(super) fn selection(sqlbuild: &[&str], dbt: &[&str]) -> DbtSelection { + DbtSelection { + sqlbuild_model_names: texts(sqlbuild), + dbt_required_unique_ids: texts(dbt), + ..DbtSelection::default() + } +} diff --git a/crates/sqlbuild-core/src/dbt_graph/tests/test_dbt_graph.rs b/crates/sqlbuild-core/src/dbt_graph/tests/test_dbt_graph.rs new file mode 100644 index 0000000000..a3e14579b8 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/tests/test_dbt_graph.rs @@ -0,0 +1,202 @@ +use crate::dbt_graph::errors::DbtSelectionError; +use crate::dbt_graph::main::dbt_downstream_sqlbuild_models::dbt_downstream_sqlbuild_models; +use crate::dbt_graph::main::dbt_fqn_selector_term::dbt_fqn_selector_term; +use crate::dbt_graph::models::DbtSelection; +use crate::dbt_graph::tests::helpers::{graph, select, selection, stable_edges}; +use crate::dbt_graph::tests::test_types::{GraphTestCase, SelectionTestCase, TextTestCase}; + +#[test] +fn given_manifest_and_project_when_building_graph_then_sorts_edges_in_node_order() { + let test_cases = [ + GraphTestCase { + description: "upstream edges keep duplicates and drop unknown dependencies", + edges: stable_edges(&graph().upstream), + expected_edges: vec![ + ( + "dbt:model:model.a.stg", + vec!["dbt:source:seed.a.codes", "dbt:source:source.a.raw.orders"], + ), + ("dbt:source:source.a.raw.orders", vec![]), + ("dbt:source:seed.a.codes", vec![]), + ( + "dbt:model:model.a.int", + vec!["dbt:model:model.a.stg", "dbt:model:model.a.stg"], + ), + ("dbt:model:model.a.dup", vec![]), + ("dbt:model:model.b.dup", vec![]), + ("sqb:model:fact", vec!["dbt:model:model.a.int"]), + ("sqb:model:mart", vec!["sqb:model:fact"]), + ("sqb:model:local", vec![]), + ], + }, + GraphTestCase { + description: "downstream edges invert upstream edges", + edges: stable_edges(&graph().downstream), + expected_edges: vec![ + ( + "dbt:model:model.a.stg", + vec!["dbt:model:model.a.int", "dbt:model:model.a.int"], + ), + ( + "dbt:source:source.a.raw.orders", + vec!["dbt:model:model.a.stg"], + ), + ("dbt:source:seed.a.codes", vec!["dbt:model:model.a.stg"]), + ("dbt:model:model.a.int", vec!["sqb:model:fact"]), + ("dbt:model:model.a.dup", vec![]), + ("dbt:model:model.b.dup", vec![]), + ("sqb:model:fact", vec!["sqb:model:mart"]), + ("sqb:model:mart", vec![]), + ("sqb:model:local", vec![]), + ], + }, + ]; + + for test_case in test_cases { + let edges: Vec<(&str, Vec<&str>)> = test_case + .edges + .iter() + .map(|(key, targets)| (key.as_str(), targets.iter().map(String::as_str).collect())) + .collect(); + assert_eq!(edges, test_case.expected_edges, "{}", test_case.description); + } +} + +#[test] +fn given_selectors_when_resolving_then_returns_models_dbt_nodes_and_anchors() { + let test_cases = [ + SelectionTestCase { + description: "upstream expansion requires dbt models and sources", + select: vec!["+fact"], + exclude: vec![], + anchors: vec![], + expected_selection: Ok(selection( + &["fact"], + &[ + "model.a.int", + "model.a.stg", + "seed.a.codes", + "source.a.raw.orders", + ], + )), + }, + SelectionTestCase { + description: "downstream expansion then exclusion", + select: vec!["fact+"], + exclude: vec!["mart+"], + anchors: vec![], + expected_selection: Ok(selection(&["fact"], &[])), + }, + SelectionTestCase { + description: "dbt to SQLBuild path with downstream expansion", + select: vec!["stg ~ fact+"], + exclude: vec![], + anchors: vec![], + expected_selection: Ok(selection( + &["fact", "mart"], + &["model.a.int", "model.a.stg"], + )), + }, + SelectionTestCase { + description: "a path with no route selects nothing", + select: vec!["local~fact"], + exclude: vec![], + anchors: vec![], + expected_selection: Ok(selection(&[], &[])), + }, + SelectionTestCase { + description: "trailing-plus dbt anchors record every term in order", + select: vec!["tag:x+", "state:modified", "tag:x+", "path:models\\marts"], + exclude: vec![], + anchors: vec![("tag:x+", vec!["model.a.stg"])], + expected_selection: Ok(DbtSelection { + dbt_anchor_terms: vec!["tag:x+".to_owned(), "tag:x+".to_owned()], + dbt_anchor_unique_ids_by_term: vec![( + "tag:x+".to_owned(), + vec!["model.a.stg".to_owned()], + )], + path_translations: vec![( + "path:models\\marts".to_owned(), + "path:models/marts".to_owned(), + )], + ..selection(&["fact", "mart"], &["model.a.int"]) + }), + }, + SelectionTestCase { + description: "an ambiguous dbt endpoint is a path selector error", + select: vec!["dup~fact"], + exclude: vec![], + anchors: vec![], + expected_selection: Err(DbtSelectionError::PathSelector( + "path selector 'dup~fact' endpoint 'dup' is ambiguous across dbt models".to_owned(), + )), + }, + SelectionTestCase { + description: "an unknown endpoint is a path selector error", + select: vec!["fact~nope"], + exclude: vec![], + anchors: vec![], + expected_selection: Err(DbtSelectionError::PathSelector( + "path selector 'fact~nope' endpoint 'nope' is not a known SQLBuild or dbt model" + .to_owned(), + )), + }, + SelectionTestCase { + description: "a missing endpoint name is a path selector error", + select: vec!["fact~ "], + exclude: vec![], + anchors: vec![], + expected_selection: Err(DbtSelectionError::PathSelector( + "path selector 'fact~ ' requires names on both sides of '~'".to_owned(), + )), + }, + SelectionTestCase { + description: "resolver failures pass through", + select: vec!["++"], + exclude: vec![], + anchors: vec![], + expected_selection: Err(DbtSelectionError::Resolver( + "selector '++' has no name".to_owned(), + )), + }, + ]; + + for test_case in test_cases { + assert_eq!( + select(&test_case.select, &test_case.exclude, &test_case.anchors), + test_case.expected_selection, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_dbt_nodes_when_rendering_terms_then_returns_names() { + let test_cases = [ + TextTestCase { + description: "SQLBuild models downstream of dbt models", + rendered: dbt_downstream_sqlbuild_models( + &graph(), + &["model.a.stg".to_owned(), "model.unknown".to_owned()], + ), + expected_text: vec!["fact", "mart"], + }, + TextTestCase { + description: "FQN selector terms and their fallback", + rendered: vec![ + dbt_fqn_selector_term(&["a".to_owned(), "orders".to_owned()], "orders"), + dbt_fqn_selector_term(&[], "orders"), + ], + expected_text: vec!["fqn:a.orders", "orders"], + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_case.rendered, test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/dbt_graph/tests/test_types.rs b/crates/sqlbuild-core/src/dbt_graph/tests/test_types.rs new file mode 100644 index 0000000000..7f3f0af0f3 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/tests/test_types.rs @@ -0,0 +1,26 @@ +use crate::dbt_graph::errors::DbtSelectionError; +use crate::dbt_graph::models::DbtSelection; + +pub(super) struct GraphTestCase { + pub(super) description: &'static str, + pub(super) edges: Vec<(String, Vec)>, + pub(super) expected_edges: Vec<(&'static str, Vec<&'static str>)>, +} + +pub(super) struct SelectionTestCase { + pub(super) description: &'static str, + pub(super) select: Vec<&'static str>, + pub(super) exclude: Vec<&'static str>, + pub(super) anchors: Vec<(&'static str, Vec<&'static str>)>, + pub(super) expected_selection: Result>, +} + +pub(super) struct TextTestCase { + pub(super) description: &'static str, + pub(super) rendered: Vec, + pub(super) expected_text: Vec<&'static str>, +} + +pub(super) struct NameResolver { + pub(super) names: Vec<&'static str>, +} diff --git a/crates/sqlbuild-core/src/dbt_graph/types.rs b/crates/sqlbuild-core/src/dbt_graph/types.rs new file mode 100644 index 0000000000..8f128cb566 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_graph/types.rs @@ -0,0 +1,13 @@ +//! Caller-supplied SQLBuild selector behaviour. + +use crate::dbt_graph::models::DbtSelectorExpansion; + +/// The caller's SQLBuild selector parsing and model name resolution. +pub trait DbtSelectorResolver { + /// The caller's failure type. + type Error; + /// Split `+` expansion markers from one selector term. + fn split(&self, term: &str) -> Result; + /// SQLBuild model names one selector core matches, and its path translation. + fn model_names(&self, core: &str) -> Result<(Vec, Option), Self::Error>; +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index d3d66a0ae7..83e662bc9d 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -13,6 +13,7 @@ pub mod cost_command; pub mod cost_report; pub mod dbt_command; pub mod dbt_events; +pub mod dbt_graph; pub mod debug_report; pub mod format_command; pub mod freshness_text; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs new file mode 100644 index 0000000000..7f332e870a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs @@ -0,0 +1,343 @@ +//! The combined dbt and SQLBuild graph, `sqb dbt` selector resolution and dbt selector terms. + +use std::collections::{HashMap, HashSet}; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_core::dbt_graph::constants::PATH_SELECTOR_ERROR_CODE; +use sqlbuild_core::dbt_graph::errors::DbtSelectionError; +use sqlbuild_core::dbt_graph::main::build_dbt_combined_graph::build_dbt_combined_graph as build_graph; +use sqlbuild_core::dbt_graph::main::dbt_downstream_sqlbuild_models::dbt_downstream_sqlbuild_models; +use sqlbuild_core::dbt_graph::main::dbt_fqn_selector_term::dbt_fqn_selector_term as fqn_selector_term; +use sqlbuild_core::dbt_graph::main::resolve_dbt_selection::resolve_dbt_selection; +use sqlbuild_core::dbt_graph::models::{ + DbtCombinedGraph as CombinedGraph, DbtGraphEdges, DbtGraphManifest, DbtGraphManifestModel, + DbtGraphProjectModel, DbtSelection, DbtSelectionRequest, DbtSelectorExpansion, +}; +use sqlbuild_core::dbt_graph::types::DbtSelectorResolver; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::dbt_interop::manifest::resolve_sqlbuild_model_dbt_refs; + +const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; +const DBT_EXCEPTIONS: &str = "sqlbuild.integrations.dbt.exceptions"; +const SELECTOR_EXPANSION_MODULE: &str = + "sqlbuild.compiler.planner.main.selection.selector_expansion"; +const MODEL_SELECTORS_MODULE: &str = + "sqlbuild.compiler.planner.main.selection.sqlbuild_model_selectors"; +const MODEL_RESOURCE_TYPE: &str = "model"; +const SELECTION_OPTIONS: (&[&str], &[&str]) = ( + &["project", "graph", "select"], + &["manifest", "exclude", "dbt_anchor_unique_ids_by_term"], +); + +/// The combined downstream-only dbt and SQLBuild graph, keyed by `owner:type:name` stable IDs. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct DbtCombinedGraph { + graph: CombinedGraph, +} + +fn stable_edges<'py>(py: Python<'py>, edges: &DbtGraphEdges) -> PyResult> { + let result: Bound<'py, PyDict> = PyDict::new(py); + for key in &edges.order { + let targets: Vec = edges + .targets + .get(key) + .into_iter() + .flatten() + .map(|target| target.stable_id()) + .collect(); + result.set_item(key.stable_id(), PyTuple::new(py, targets)?)?; + } + Ok(result) +} + +#[pymethods] +impl DbtCombinedGraph { + /// Every node's stable ID. + #[getter] + fn nodes<'py>(&self, py: Python<'py>) -> PyResult> { + PyFrozenSet::new( + py, + self.graph.upstream.order.iter().map(|key| key.stable_id()), + ) + } + + /// Upstream edges by stable ID. + #[getter] + fn upstream_deps<'py>(&self, py: Python<'py>) -> PyResult> { + stable_edges(py, &self.graph.upstream) + } + + /// Downstream edges by stable ID. + #[getter] + fn downstream_deps<'py>(&self, py: Python<'py>) -> PyResult> { + stable_edges(py, &self.graph.downstream) + } + + /// Sorted SQLBuild model names transitively downstream of any of the dbt models. + fn downstream_sqlbuild_model_names<'py>( + &self, + py: Python<'py>, + unique_ids: &Bound<'py, PyAny>, + ) -> PyResult> { + let unique_ids: Vec = texts(unique_ids)?; + PyTuple::new(py, dbt_downstream_sqlbuild_models(&self.graph, &unique_ids)) + } +} + +fn texts(values: &Bound<'_, PyAny>) -> PyResult> { + values.try_iter()?.map(|value| value?.extract()).collect() +} + +fn key_set(mapping: &Bound<'_, PyAny>) -> PyResult> { + Ok(texts(mapping)?.into_iter().collect()) +} + +fn graph_manifest(manifest: &Bound<'_, PyAny>) -> PyResult { + let mut models: Vec = Vec::new(); + for model in manifest + .getattr("models_by_unique_id")? + .call_method0("values")? + .try_iter()? + { + let model: Bound<'_, PyAny> = model?; + models.push(DbtGraphManifestModel { + unique_id: model.getattr("unique_id")?.extract()?, + name: model.getattr("name")?.extract()?, + depends_on_nodes: texts(&model.getattr("depends_on_nodes")?)?, + }); + } + Ok(DbtGraphManifest { + models, + source_ids: key_set(&manifest.getattr("sources_by_unique_id")?)?, + seed_ids: key_set(&manifest.getattr("seeds_by_unique_id")?)?, + }) +} + +fn project_models<'py>( + py: Python<'py>, + project: &Bound<'py, PyAny>, + manifest: &Bound<'py, PyAny>, +) -> PyResult> { + let mut refs_by_name: HashMap> = HashMap::new(); + for pair in + resolve_sqlbuild_model_dbt_refs(py, project.clone(), manifest.clone(), None)?.try_iter()? + { + let (model, dbt_model): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; + refs_by_name + .entry(model.getattr("name")?.extract()?) + .or_default() + .push(dbt_model.getattr("unique_id")?.extract()?); + } + let mut models: Vec = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let name: String = model.getattr("name")?.extract()?; + let mut model_deps: Vec = Vec::new(); + for dependency in model.getattr("deps")?.try_iter()? { + let dependency: Bound<'py, PyAny> = dependency?; + if dependency + .getattr("resource_type")? + .eq(MODEL_RESOURCE_TYPE)? + { + model_deps.push(dependency.getattr("name")?.extract()?); + } + } + let dbt_ref_unique_ids: Vec = refs_by_name.get(&name).cloned().unwrap_or_default(); + models.push(DbtGraphProjectModel { + name, + model_deps, + dbt_ref_unique_ids, + }); + } + Ok(models) +} + +/// Build the combined graph from a dbt manifest index and a compiled SQLBuild project. +#[pyfunction] +#[pyo3(signature = (*, manifest, project))] +fn build_dbt_combined_graph<'py>( + py: Python<'py>, + manifest: Bound<'py, PyAny>, + project: Bound<'py, PyAny>, +) -> PyResult { + let models: Vec = project_models(py, &project, &manifest)?; + Ok(DbtCombinedGraph { + graph: build_graph(&graph_manifest(&manifest)?, &models), + }) +} + +struct PlannerResolver<'py> { + project: Bound<'py, PyAny>, + split: Bound<'py, PyAny>, + resolve: Bound<'py, PyAny>, +} + +impl DbtSelectorResolver for PlannerResolver<'_> { + type Error = PyErr; + + fn split(&self, term: &str) -> PyResult { + let parsed: Bound<'_, PyAny> = self.split.call1((term,))?; + Ok(DbtSelectorExpansion { + core: parsed.getattr("core")?.extract()?, + upstream: parsed.getattr("upstream")?.is_truthy()?, + downstream: parsed.getattr("downstream")?.is_truthy()?, + }) + } + + fn model_names(&self, core: &str) -> PyResult<(Vec, Option)> { + let py: Python<'_> = self.project.py(); + let (names, translated): (Bound<'_, PyAny>, Option) = call_with( + &self.resolve, + &[ + ("project", self.project.clone()), + ("term", PyString::new(py, core).into_any()), + ], + )? + .extract()?; + Ok((texts(&names)?, translated)) + } +} + +fn selection_error(py: Python<'_>, error: DbtSelectionError) -> PyErr { + match error { + DbtSelectionError::Resolver(error) => error, + DbtSelectionError::PathSelector(message) => { + let raised = || -> PyResult { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("code", PATH_SELECTOR_ERROR_CODE)?; + let error: Bound<'_, PyAny> = + imported(py, DBT_EXCEPTIONS, "DbtInteropArgumentError")? + .call((message,), Some(&keywords))?; + Ok(PyErr::from_value(error)) + }; + raised().unwrap_or_else(|failure| failure) + } + } +} + +fn anchors(mapping: &Bound<'_, PyAny>) -> PyResult>> { + let mut result: HashMap> = HashMap::new(); + if mapping.is_none() { + return Ok(result); + } + for item in mapping.call_method0("items")?.try_iter()? { + let (term, unique_ids): (String, Bound<'_, PyAny>) = item?.extract()?; + result.insert(term, texts(&unique_ids)?); + } + Ok(result) +} + +fn manifest_models(manifest: &Bound<'_, PyAny>) -> PyResult>> { + if manifest.is_none() { + return Ok(None); + } + let models: Vec<(String, String)> = graph_manifest(manifest)? + .models + .into_iter() + .map(|model| (model.unique_id, model.name)) + .collect(); + Ok(Some(models)) +} + +fn selection_result<'py>(py: Python<'py>, selection: DbtSelection) -> PyResult> { + let anchors_by_term: Bound<'py, PyDict> = PyDict::new(py); + for (term, unique_ids) in selection.dbt_anchor_unique_ids_by_term { + anchors_by_term.set_item(term, PyTuple::new(py, unique_ids)?)?; + } + let translations: Vec> = selection + .path_translations + .into_iter() + .map(|(term, translated)| PyTuple::new(py, [term, translated])) + .collect::>()?; + call_with( + &imported(py, DBT_MODELS, "DbtInteropSelectionResult")?, + &[ + ( + "sqlbuild_model_names", + PyTuple::new(py, selection.sqlbuild_model_names)?.into_any(), + ), + ( + "dbt_required_unique_ids", + PyTuple::new(py, selection.dbt_required_unique_ids)?.into_any(), + ), + ( + "dbt_anchor_terms", + PyTuple::new(py, selection.dbt_anchor_terms)?.into_any(), + ), + ("dbt_anchor_unique_ids_by_term", anchors_by_term.into_any()), + ( + "path_translations", + PyTuple::new(py, translations)?.into_any(), + ), + ], + ) +} + +/// Resolve selected SQLBuild models and required dbt nodes from `sqb dbt` selectors. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_dbt_interop_sqlbuild_selection<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_dbt_interop_sqlbuild_selection", + options, + SELECTION_OPTIONS, + )?; + let manifest_models: Option> = + manifest_models(&arguments.value("manifest")?)?; + let anchor_unique_ids_by_term: HashMap> = + anchors(&arguments.value("dbt_anchor_unique_ids_by_term")?)?; + let select: Vec = texts(&arguments.value("select")?)?; + let exclude_terms: Bound<'py, PyAny> = arguments.value("exclude")?; + let exclude: Vec = if exclude_terms.is_none() { + Vec::new() + } else { + texts(&exclude_terms)? + }; + let request: DbtSelectionRequest<'_> = DbtSelectionRequest { + select: &select, + exclude: &exclude, + anchor_unique_ids_by_term: &anchor_unique_ids_by_term, + manifest_models: manifest_models.as_deref(), + }; + let graph: Bound<'py, DbtCombinedGraph> = arguments.value("graph")?.cast_into()?; + let resolver: PlannerResolver<'py> = PlannerResolver { + project: arguments.value("project")?, + split: imported(py, SELECTOR_EXPANSION_MODULE, "split_selector_expansion")?, + resolve: imported( + py, + MODEL_SELECTORS_MODULE, + "resolve_sqlbuild_model_selector_names", + )?, + }; + let selection: DbtSelection = resolve_dbt_selection(&resolver, &graph.get().graph, &request) + .map_err(|error| selection_error(py, error))?; + selection_result(py, selection) +} + +/// The exact dbt FQN selector when the node has an FQN, else `fallback`. +#[pyfunction] +#[pyo3(signature = (*, fqn, fallback))] +fn dbt_fqn_selector_term(fqn: &Bound<'_, PyAny>, fallback: &str) -> PyResult { + Ok(fqn_selector_term(&texts(fqn)?, fallback)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(build_dbt_combined_graph, module)?)?; + module.add_function(wrap_pyfunction!( + resolve_dbt_interop_sqlbuild_selection, + module + )?)?; + module.add_function(wrap_pyfunction!(dbt_fqn_selector_term, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs index 31f7a842d3..c31414eccb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs @@ -525,7 +525,7 @@ fn name_set(names: &Bound<'_, PyAny>) -> PyResult> { /// Return resolved dbt refs used directly by SQLBuild models. #[pyfunction] #[pyo3(signature = (*, project, manifest, selected_model_names = None))] -fn resolve_sqlbuild_model_dbt_refs<'py>( +pub(crate) fn resolve_sqlbuild_model_dbt_refs<'py>( py: Python<'py>, project: Bound<'py, PyAny>, manifest: Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs index 40ad34c6e3..e05c3f9b53 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod graph; pub(crate) mod manifest; pub(crate) mod plan; pub(crate) mod profile; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs index 2cd3a2ed8b..4a673eec06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::dbt_interop::{manifest, plan, profile}; +use crate::bindings::_helpers::dbt_interop::{graph, manifest, plan, profile}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + graph::register(module)?; manifest::register(module)?; plan::register(module)?; profile::register(module) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index d0c7f72284..b8accc4be4 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1460,6 +1460,29 @@ def resolve_sqlbuild_model_dbt_refs( manifest: DbtManifestIndex, selected_model_names: Iterable[str] | None = None, ) -> tuple[tuple[CompiledModelView, DbtManifestModel], ...]: ... + +class DbtCombinedGraph: + @property + def nodes(self) -> frozenset[str]: ... + @property + def upstream_deps(self) -> dict[str, tuple[str, ...]]: ... + @property + def downstream_deps(self) -> dict[str, tuple[str, ...]]: ... + def downstream_sqlbuild_model_names(self, unique_ids: Iterable[str]) -> tuple[str, ...]: ... + +def build_dbt_combined_graph( + *, manifest: DbtManifestIndex, project: CompiledProjectView +) -> DbtCombinedGraph: ... +def resolve_dbt_interop_sqlbuild_selection( + *, + project: CompiledProjectView, + graph: DbtCombinedGraph, + select: Sequence[str], + manifest: DbtManifestIndex | None = None, + exclude: Sequence[str] | None = None, + dbt_anchor_unique_ids_by_term: Mapping[str, Sequence[str]] | None = None, +) -> DbtInteropSelectionResult: ... +def dbt_fqn_selector_term(*, fqn: Sequence[str], fallback: str) -> str: ... def resolve_dbt_reference_relation( *, manifest: DbtManifestIndex | None, diff --git a/src/sqlbuild/integrations/dbt/_helpers/graph/core.py b/src/sqlbuild/integrations/dbt/_helpers/graph/core.py deleted file mode 100644 index 3a6ff7d900..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/graph/core.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Combined dbt and SQLBuild graph helpers.""" - -from __future__ import annotations - -from sqlbuild._native import invert_edges, resolve_sqlbuild_model_dbt_refs, transitive_closure -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.integrations.dbt.models import ( - DbtCombinedGraph, - DbtCombinedGraphKey, - DbtManifestIndex, - DbtManifestModel, - DbtManifestSource, -) -from sqlbuild.integrations.dbt.types import DbtCombinedGraphOwner, DbtCombinedGraphResourceType - - -def build_dbt_combined_graph( - *, manifest: DbtManifestIndex, project: CompiledProjectView -) -> DbtCombinedGraph: - """Build a downstream-only combined graph from dbt manifest and SQLBuild compile.""" - - upstream: dict[DbtCombinedGraphKey, list[DbtCombinedGraphKey]] = {} - upstream = _add_dbt_model_edges(upstream=upstream, manifest=manifest) - upstream = _add_sqlbuild_model_edges(upstream=upstream, manifest=manifest, project=project) - - normalized_upstream: dict[DbtCombinedGraphKey, tuple[DbtCombinedGraphKey, ...]] = { - key: _sorted_keys(deps) for key, deps in upstream.items() - } - downstream: dict[DbtCombinedGraphKey, tuple[DbtCombinedGraphKey, ...]] = ( - build_combined_downstream_deps(normalized_upstream) - ) - return DbtCombinedGraph( - nodes=frozenset(normalized_upstream), - upstream_deps=normalized_upstream, - downstream_deps=downstream, - ) - - -def dbt_model_graph_key(unique_id: str) -> DbtCombinedGraphKey: - """Return the combined graph key for one dbt model unique_id.""" - - return DbtCombinedGraphKey( - owner=DbtCombinedGraphOwner.DBT, - resource_type=DbtCombinedGraphResourceType.MODEL, - name=unique_id, - ) - - -def sqlbuild_model_graph_key(model_name: str) -> DbtCombinedGraphKey: - """Return the combined graph key for one SQLBuild model name.""" - - return DbtCombinedGraphKey( - owner=DbtCombinedGraphOwner.SQLBUILD, - resource_type=DbtCombinedGraphResourceType.MODEL, - name=model_name, - ) - - -def dbt_source_graph_key(unique_id: str) -> DbtCombinedGraphKey: - """Return the combined graph key for one dbt source unique_id.""" - - return DbtCombinedGraphKey( - owner=DbtCombinedGraphOwner.DBT, - resource_type=DbtCombinedGraphResourceType.SOURCE, - name=unique_id, - ) - - -def build_combined_downstream_deps( - upstream: dict[DbtCombinedGraphKey, tuple[DbtCombinedGraphKey, ...]], -) -> dict[DbtCombinedGraphKey, tuple[DbtCombinedGraphKey, ...]]: - """Return downstream edges keyed by upstream combined graph key.""" - - inverted: dict[DbtCombinedGraphKey, tuple[DbtCombinedGraphKey, ...]] = invert_edges( - edges=upstream - ) - return {key: _sorted_keys(list(values)) for key, values in inverted.items()} - - -def expand_combined_upstream( - *, - key: DbtCombinedGraphKey, - upstream: dict[DbtCombinedGraphKey, tuple[DbtCombinedGraphKey, ...]], -) -> frozenset[DbtCombinedGraphKey]: - """Return all transitive upstream combined graph keys.""" - - return transitive_closure(start=key, edges=upstream) - - -def expand_combined_downstream( - *, - key: DbtCombinedGraphKey, - downstream: dict[DbtCombinedGraphKey, tuple[DbtCombinedGraphKey, ...]], -) -> frozenset[DbtCombinedGraphKey]: - """Return all transitive downstream combined graph keys.""" - - return transitive_closure(start=key, edges=downstream) - - -def _add_dbt_model_edges( - *, upstream: dict[DbtCombinedGraphKey, list[DbtCombinedGraphKey]], manifest: DbtManifestIndex -) -> dict[DbtCombinedGraphKey, list[DbtCombinedGraphKey]]: - model: DbtManifestModel - for model in manifest.models_by_unique_id.values(): - key: DbtCombinedGraphKey = dbt_model_graph_key(model.unique_id) - upstream.setdefault(key, []) - dep_unique_id: str - for dep_unique_id in model.depends_on_nodes: - if dep_unique_id in manifest.models_by_unique_id: - upstream[key].append(dbt_model_graph_key(dep_unique_id)) - if dep_unique_id in manifest.sources_by_unique_id: - source: DbtManifestSource = manifest.sources_by_unique_id[dep_unique_id] - source_key: DbtCombinedGraphKey = dbt_source_graph_key(source.unique_id) - upstream.setdefault(source_key, []) - upstream[key].append(source_key) - if dep_unique_id in manifest.seeds_by_unique_id: - seed_key: DbtCombinedGraphKey = dbt_source_graph_key(dep_unique_id) - upstream.setdefault(seed_key, []) - upstream[key].append(seed_key) - return upstream - - -def _add_sqlbuild_model_edges( - *, - upstream: dict[DbtCombinedGraphKey, list[DbtCombinedGraphKey]], - manifest: DbtManifestIndex, - project: CompiledProjectView, -) -> dict[DbtCombinedGraphKey, list[DbtCombinedGraphKey]]: - sqlbuild_model_names: frozenset[str] = frozenset(model.name for model in project.models) - dbt_refs_by_model_name: dict[str, list[DbtManifestModel]] = {} - sqlbuild_model: CompiledModelView - dbt_model: DbtManifestModel - for sqlbuild_model, dbt_model in resolve_sqlbuild_model_dbt_refs( - project=project, - manifest=manifest, - ): - dbt_refs_by_model_name.setdefault(sqlbuild_model.name, []).append(dbt_model) - model: CompiledModelView - for model in project.models: - key: DbtCombinedGraphKey = sqlbuild_model_graph_key(model.name) - upstream.setdefault(key, []) - dep_key: CompiledObjectKey - for dep_key in model.deps: - if dep_key.resource_type != CompiledResourceType.MODEL: - continue - if dep_key.name not in sqlbuild_model_names: - continue - upstream[key].append(sqlbuild_model_graph_key(dep_key.name)) - - for dbt_model in dbt_refs_by_model_name.get(model.name, ()): - upstream[key].append(dbt_model_graph_key(dbt_model.unique_id)) - return upstream - - -def _sorted_keys(keys: list[DbtCombinedGraphKey]) -> tuple[DbtCombinedGraphKey, ...]: - return tuple(sorted(keys, key=lambda key: key.stable_id)) diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py index 84cd7de5a9..0aa494a0a4 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py @@ -8,11 +8,7 @@ from pathlib import Path from typing import TextIO, cast -from sqlbuild._native import format_summary_footer, render_dbt_node_result -from sqlbuild.integrations.dbt._helpers.graph.core import ( - dbt_model_graph_key, - expand_combined_downstream, -) +from sqlbuild._native import DbtCombinedGraph, format_summary_footer, render_dbt_node_result from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.constants import ( DBT_DEFER_FLAG, @@ -30,8 +26,6 @@ ) from sqlbuild.integrations.dbt.models import ( DbtCliOptions, - DbtCombinedGraph, - DbtCombinedGraphKey, DbtCommandExecutionResult, DbtCommandResult, DbtInteropPlan, @@ -40,12 +34,7 @@ DbtNodeExecutionResult, DbtNodeMessage, ) -from sqlbuild.integrations.dbt.types import ( - DbtCombinedGraphOwner, - DbtCombinedGraphResourceType, - DbtInteropCommand, - DbtSupportedResourceType, -) +from sqlbuild.integrations.dbt.types import DbtInteropCommand, DbtSupportedResourceType from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter @@ -187,19 +176,7 @@ def build_failed_sqlbuild_model_names( failed_unique_ids.update( manifest.validation_depends_on_nodes_by_unique_id.get(result.unique_id, ()) ) - names: set[str] = set() - for unique_id in failed_unique_ids: - downstream: frozenset[DbtCombinedGraphKey] = expand_combined_downstream( - key=dbt_model_graph_key(unique_id), - downstream=graph.downstream_deps, - ) - for key in downstream: - if ( - key.owner == DbtCombinedGraphOwner.SQLBUILD - and key.resource_type == DbtCombinedGraphResourceType.MODEL - ): - names.add(key.name) - return tuple(sorted(names)) + return graph.downstream_sqlbuild_model_names(failed_unique_ids) def render_dbt_execution_summary_footer( diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py index 20c24aa6fd..8dc4badeff 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py @@ -9,7 +9,9 @@ from typing import TextIO from sqlbuild._native import ( + DbtCombinedGraph, build_compiled_project, + build_dbt_combined_graph, discover_project_inputs, load_dbt_manifest_index, ) @@ -27,7 +29,6 @@ from sqlbuild.integrations.dbt.main.cli._route_interop_args import route_dbt_interop_args from sqlbuild.integrations.dbt.main.config._resolve_plan_options import resolve_dbt_plan_options from sqlbuild.integrations.dbt.main.config._resolve_vars_mapping import resolve_dbt_vars_mapping -from sqlbuild.integrations.dbt.main.graph._build_combined_graph import build_dbt_combined_graph from sqlbuild.integrations.dbt.main.manifest._resolve_manifest_path import ( resolve_dbt_manifest_path, ) @@ -38,7 +39,6 @@ ) from sqlbuild.integrations.dbt.models import ( DbtCliOptions, - DbtCombinedGraph, DbtCommandResult, DbtInteropCommandArgs, DbtInteropCompiledProject, diff --git a/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py b/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py index f2d0be7060..53356ac554 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py +++ b/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py @@ -4,21 +4,24 @@ from collections.abc import Sequence -from sqlbuild._native import build_dbt_interop_plan, resolve_sqlbuild_model_dbt_refs +from sqlbuild._native import ( + DbtCombinedGraph, + build_dbt_interop_plan, + dbt_fqn_selector_term, + resolve_dbt_interop_sqlbuild_selection, + resolve_sqlbuild_model_dbt_refs, +) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.main.selection.selector_expansion import split_selector_expansion from sqlbuild.compiler.planner.main.selection.sqlbuild_model_selectors import ( resolve_sqlbuild_model_selector_names, ) from sqlbuild.compiler.planner.models import SelectorExpansion -from sqlbuild.integrations.dbt._helpers.selection.core import resolve_dbt_interop_sqlbuild_selection -from sqlbuild.integrations.dbt._helpers.selection.selector_terms import dbt_fqn_selector_term from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.constants import DBT_PATH_SELECTOR_SEPARATOR from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError from sqlbuild.integrations.dbt.models import ( DbtCliOptions, - DbtCombinedGraph, DbtInteropCommandArgs, DbtInteropPlan, DbtInteropSelectionResult, diff --git a/src/sqlbuild/integrations/dbt/_helpers/selection/core.py b/src/sqlbuild/integrations/dbt/_helpers/selection/core.py deleted file mode 100644 index 9c7ff22c29..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/selection/core.py +++ /dev/null @@ -1,248 +0,0 @@ -"""SQLBuild selection resolution for future `sqb dbt` commands.""" - -from __future__ import annotations - -from collections.abc import Mapping, Sequence - -from sqlbuild._native import path_nodes -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.main.selection.selector_expansion import split_selector_expansion -from sqlbuild.compiler.planner.main.selection.sqlbuild_model_selectors import ( - resolve_sqlbuild_model_selector_names, -) -from sqlbuild.compiler.planner.models import SelectorExpansion -from sqlbuild.integrations.dbt._helpers.graph.core import ( - dbt_model_graph_key, - expand_combined_downstream, - expand_combined_upstream, - sqlbuild_model_graph_key, -) -from sqlbuild.integrations.dbt.constants import DBT_PATH_SELECTOR_SEPARATOR -from sqlbuild.integrations.dbt.exceptions import DbtInteropArgumentError -from sqlbuild.integrations.dbt.models import ( - DbtCombinedGraph, - DbtCombinedGraphKey, - DbtInteropSelectionResult, - DbtManifestIndex, - DbtManifestModel, -) -from sqlbuild.integrations.dbt.types import DbtCombinedGraphOwner - - -def resolve_dbt_interop_sqlbuild_selection( - *, - project: CompiledProjectView, - manifest: DbtManifestIndex | None = None, - graph: DbtCombinedGraph, - select: Sequence[str], - exclude: Sequence[str] = (), - dbt_anchor_unique_ids_by_term: Mapping[str, Sequence[str]] | None = None, -) -> DbtInteropSelectionResult: - """Resolve selected SQLBuild models and required dbt nodes from `sqb dbt` selectors.""" - - anchors_by_term: Mapping[str, Sequence[str]] = dbt_anchor_unique_ids_by_term or {} - selected_sqlbuild: set[str] = set() - required_dbt: set[str] = set() - anchor_terms: list[str] = [] - anchor_result: dict[str, tuple[str, ...]] = {} - path_translations: list[tuple[str, str]] = [] - - term: str - for term in select: - parsed: SelectorExpansion = split_selector_expansion(term) - path_keys: frozenset[DbtCombinedGraphKey] | None = _resolve_path_between_keys( - project=project, - manifest=manifest, - graph=graph, - parsed=parsed, - raw_term=term, - ) - if path_keys is not None: - expanded_keys: set[DbtCombinedGraphKey] = set(path_keys) - key: DbtCombinedGraphKey - if parsed.upstream: - for key in tuple(path_keys): - expanded_keys.update( - expand_combined_upstream(key=key, upstream=graph.upstream_deps) - ) - if parsed.downstream: - for key in tuple(path_keys): - expanded_keys.update( - expand_combined_downstream(key=key, downstream=graph.downstream_deps) - ) - selected_sqlbuild, required_dbt = _add_expanded_keys( - keys=frozenset(expanded_keys), - selected_sqlbuild=selected_sqlbuild, - required_dbt=required_dbt, - ) - continue - - direct_keys: frozenset[DbtCombinedGraphKey] - direct_keys, translated_path = _resolve_direct_sqlbuild_keys( - project=project, - term=parsed.core, - ) - if translated_path is not None: - path_translations.append((term, translated_path)) - if direct_keys: - expanded_keys: set[DbtCombinedGraphKey] = set(direct_keys) - key: DbtCombinedGraphKey - if parsed.upstream: - for key in tuple(direct_keys): - expanded_keys.update( - expand_combined_upstream(key=key, upstream=graph.upstream_deps) - ) - if parsed.downstream: - for key in tuple(direct_keys): - expanded_keys.update( - expand_combined_downstream(key=key, downstream=graph.downstream_deps) - ) - selected_sqlbuild, required_dbt = _add_expanded_keys( - keys=frozenset(expanded_keys), - selected_sqlbuild=selected_sqlbuild, - required_dbt=required_dbt, - ) - continue - - if not parsed.downstream: - continue - anchor_unique_ids: tuple[str, ...] = tuple(anchors_by_term.get(term, ())) - anchor_terms.append(term) - anchor_result[term] = anchor_unique_ids - for unique_id in anchor_unique_ids: - downstream_keys: frozenset[DbtCombinedGraphKey] = expand_combined_downstream( - key=dbt_model_graph_key(unique_id), downstream=graph.downstream_deps - ) - selected_sqlbuild, required_dbt = _add_expanded_keys( - keys=downstream_keys, - selected_sqlbuild=selected_sqlbuild, - required_dbt=required_dbt, - ) - - excluded_sqlbuild: frozenset[str] = _resolve_excluded_sqlbuild_names( - project=project, - exclude=exclude, - ) - selected_sqlbuild.difference_update(excluded_sqlbuild) - return DbtInteropSelectionResult( - sqlbuild_model_names=tuple(sorted(selected_sqlbuild)), - dbt_required_unique_ids=tuple(sorted(required_dbt)), - dbt_anchor_terms=tuple(anchor_terms), - dbt_anchor_unique_ids_by_term=anchor_result, - path_translations=tuple(path_translations), - ) - - -def _resolve_path_between_keys( - *, - project: CompiledProjectView, - manifest: DbtManifestIndex | None, - graph: DbtCombinedGraph, - parsed: SelectorExpansion, - raw_term: str, -) -> frozenset[DbtCombinedGraphKey] | None: - if DBT_PATH_SELECTOR_SEPARATOR not in parsed.core: - return None - start_name, end_name = _path_endpoint_names(term=parsed.core, raw_term=raw_term) - start_key: DbtCombinedGraphKey = _resolve_path_endpoint_key( - project=project, - manifest=manifest, - endpoint=start_name, - raw_term=raw_term, - ) - end_key: DbtCombinedGraphKey = _resolve_path_endpoint_key( - project=project, - manifest=manifest, - endpoint=end_name, - raw_term=raw_term, - ) - return path_nodes(start=start_key, end=end_key, downstream=graph.downstream_deps) or frozenset() - - -def _path_endpoint_names(*, term: str, raw_term: str) -> tuple[str, str]: - start_name, end_name = (part.strip() for part in term.split(DBT_PATH_SELECTOR_SEPARATOR, 1)) - if not start_name or not end_name: - raise DbtInteropArgumentError( - f"path selector '{raw_term}' requires names on both sides of '~'", - code="C237", - ) - return start_name, end_name - - -def _resolve_path_endpoint_key( - *, - project: CompiledProjectView, - manifest: DbtManifestIndex | None, - endpoint: str, - raw_term: str, -) -> DbtCombinedGraphKey: - sqlbuild_model_names, _translation = resolve_sqlbuild_model_selector_names( - project=project, - term=endpoint, - ) - if sqlbuild_model_names: - return sqlbuild_model_graph_key(sqlbuild_model_names[0]) - if manifest is None: - raise DbtInteropArgumentError( - f"path selector '{raw_term}' endpoint '{endpoint}' is not a SQLBuild model", - code="C237", - ) - dbt_models: tuple[DbtManifestModel, ...] = tuple( - model for model in manifest.models_by_unique_id.values() if model.name == endpoint - ) - if len(dbt_models) == 1: - return dbt_model_graph_key(dbt_models[0].unique_id) - if len(dbt_models) > 1: - raise DbtInteropArgumentError( - f"path selector '{raw_term}' endpoint '{endpoint}' is ambiguous across dbt models", - code="C237", - ) - raise DbtInteropArgumentError( - f"path selector '{raw_term}' endpoint '{endpoint}' is not a known SQLBuild or dbt model", - code="C237", - ) - - -def _resolve_direct_sqlbuild_keys( - *, project: CompiledProjectView, term: str -) -> tuple[frozenset[DbtCombinedGraphKey], str | None]: - model_names, translated_path = resolve_sqlbuild_model_selector_names( - project=project, - term=term, - ) - return frozenset( - sqlbuild_model_graph_key(model_name) for model_name in model_names - ), translated_path - - -def _add_expanded_keys( - *, - keys: frozenset[DbtCombinedGraphKey], - selected_sqlbuild: set[str], - required_dbt: set[str], -) -> tuple[set[str], set[str]]: - key: DbtCombinedGraphKey - for key in keys: - if key.owner == DbtCombinedGraphOwner.SQLBUILD: - selected_sqlbuild.add(key.name) - elif key.owner == DbtCombinedGraphOwner.DBT: - required_dbt.add(key.name) - return selected_sqlbuild, required_dbt - - -def _resolve_excluded_sqlbuild_names( - *, project: CompiledProjectView, exclude: Sequence[str] -) -> frozenset[str]: - excluded: set[str] = set() - term: str - for term in exclude: - parsed: SelectorExpansion = split_selector_expansion(term) - keys, _translation = _resolve_direct_sqlbuild_keys( - project=project, - term=parsed.core, - ) - key: DbtCombinedGraphKey - for key in keys: - if key.owner == DbtCombinedGraphOwner.SQLBUILD: - excluded.add(key.name) - return frozenset(excluded) diff --git a/src/sqlbuild/integrations/dbt/_helpers/selection/selector_terms.py b/src/sqlbuild/integrations/dbt/_helpers/selection/selector_terms.py deleted file mode 100644 index edb5420ec4..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/selection/selector_terms.py +++ /dev/null @@ -1,11 +0,0 @@ -"""dbt selector term helpers.""" - -from __future__ import annotations - - -def dbt_fqn_selector_term(*, fqn: tuple[str, ...], fallback: str) -> str: - """Return the exact dbt FQN selector when available.""" - - if fqn: - return f"fqn:{'.'.join(fqn)}" - return fallback diff --git a/src/sqlbuild/integrations/dbt/main/graph/__init__.py b/src/sqlbuild/integrations/dbt/main/graph/__init__.py deleted file mode 100644 index 3df8ec0dcc..0000000000 --- a/src/sqlbuild/integrations/dbt/main/graph/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""dbt graph public operations.""" diff --git a/src/sqlbuild/integrations/dbt/main/graph/_build_combined_graph.py b/src/sqlbuild/integrations/dbt/main/graph/_build_combined_graph.py deleted file mode 100644 index de93d318d6..0000000000 --- a/src/sqlbuild/integrations/dbt/main/graph/_build_combined_graph.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Build the combined dbt and SQLBuild graph.""" - -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.integrations.dbt._helpers.graph.core import build_dbt_combined_graph as _build -from sqlbuild.integrations.dbt.models import DbtCombinedGraph, DbtManifestIndex - - -def build_dbt_combined_graph( - *, manifest: DbtManifestIndex, project: CompiledProjectView -) -> DbtCombinedGraph: - """Build the combined dbt and SQLBuild graph.""" - - return _build(manifest=manifest, project=project) diff --git a/src/sqlbuild/integrations/dbt/main/graph/_dbt_model_graph_key.py b/src/sqlbuild/integrations/dbt/main/graph/_dbt_model_graph_key.py deleted file mode 100644 index c42ad940b2..0000000000 --- a/src/sqlbuild/integrations/dbt/main/graph/_dbt_model_graph_key.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Build a dbt model graph key.""" - -from sqlbuild.integrations.dbt._helpers.graph.core import dbt_model_graph_key as _build -from sqlbuild.integrations.dbt.models import DbtCombinedGraphKey - - -def dbt_model_graph_key(unique_id: str) -> DbtCombinedGraphKey: - """Return the combined graph key for a dbt model.""" - - return _build(unique_id) diff --git a/src/sqlbuild/integrations/dbt/main/graph/_sqlbuild_model_graph_key.py b/src/sqlbuild/integrations/dbt/main/graph/_sqlbuild_model_graph_key.py deleted file mode 100644 index 350cad113a..0000000000 --- a/src/sqlbuild/integrations/dbt/main/graph/_sqlbuild_model_graph_key.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Build a SQLBuild model graph key.""" - -from sqlbuild.integrations.dbt._helpers.graph.core import sqlbuild_model_graph_key as _build -from sqlbuild.integrations.dbt.models import DbtCombinedGraphKey - - -def sqlbuild_model_graph_key(model_name: str) -> DbtCombinedGraphKey: - """Return the combined graph key for a SQLBuild model.""" - - return _build(model_name) diff --git a/src/sqlbuild/integrations/dbt/main/planning/_plan_interop_command.py b/src/sqlbuild/integrations/dbt/main/planning/_plan_interop_command.py index 4df96a516e..aab36b7088 100644 --- a/src/sqlbuild/integrations/dbt/main/planning/_plan_interop_command.py +++ b/src/sqlbuild/integrations/dbt/main/planning/_plan_interop_command.py @@ -1,5 +1,6 @@ """Plan a dbt interop command.""" +from sqlbuild._native import DbtCombinedGraph from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.integrations.dbt._helpers.planning.orchestration import ( plan_dbt_interop_command as _plan, @@ -7,7 +8,6 @@ from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.models import ( DbtCliOptions, - DbtCombinedGraph, DbtInteropCommandArgs, DbtInteropPlan, DbtManifestIndex, diff --git a/src/sqlbuild/integrations/dbt/main/selection/__init__.py b/src/sqlbuild/integrations/dbt/main/selection/__init__.py deleted file mode 100644 index 7e8f9ff6f1..0000000000 --- a/src/sqlbuild/integrations/dbt/main/selection/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""dbt selection public operations.""" diff --git a/src/sqlbuild/integrations/dbt/main/selection/_dbt_fqn_selector_term.py b/src/sqlbuild/integrations/dbt/main/selection/_dbt_fqn_selector_term.py deleted file mode 100644 index f7d2c3691d..0000000000 --- a/src/sqlbuild/integrations/dbt/main/selection/_dbt_fqn_selector_term.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Build a dbt FQN selector term.""" - -from sqlbuild.integrations.dbt._helpers.selection.selector_terms import ( - dbt_fqn_selector_term as _build, -) - - -def dbt_fqn_selector_term(*, fqn: tuple[str, ...], fallback: str) -> str: - """Return the most specific stable dbt selector term.""" - - return _build(fqn=fqn, fallback=fallback) diff --git a/src/sqlbuild/integrations/dbt/models.py b/src/sqlbuild/integrations/dbt/models.py index 8873fed831..4418aa8aa2 100644 --- a/src/sqlbuild/integrations/dbt/models.py +++ b/src/sqlbuild/integrations/dbt/models.py @@ -7,17 +7,12 @@ from pathlib import Path from typing import Any, TextIO +from sqlbuild._native import DbtCombinedGraph, dbt_fqn_selector_term from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.integrations.dbt._helpers.selection.selector_terms import dbt_fqn_selector_term -from sqlbuild.integrations.dbt.types import ( - DbtCombinedGraphOwner, - DbtCombinedGraphResourceType, - DbtInteropCommand, - DbtInteropSkipReason, -) +from sqlbuild.integrations.dbt.types import DbtInteropCommand, DbtInteropSkipReason @dataclass(frozen=True) @@ -291,43 +286,6 @@ class DbtLsResult: command: DbtCommandResult -@dataclass(frozen=True, order=True) -class DbtCombinedGraphKey: - """Stable owner-qualified key for one combined dbt/SQLBuild graph node.""" - - owner: DbtCombinedGraphOwner - resource_type: DbtCombinedGraphResourceType - name: str - - def __post_init__(self) -> None: - from sqlbuild.integrations.dbt.types import ( - DbtCombinedGraphOwner, - DbtCombinedGraphResourceType, - ) - - object.__setattr__(self, "owner", DbtCombinedGraphOwner(self.owner)) - object.__setattr__( - self, - "resource_type", - DbtCombinedGraphResourceType(self.resource_type), - ) - - @property - def stable_id(self) -> str: - """Return a stable string form for plan JSON and diagnostics.""" - - return f"{self.owner}:{self.resource_type}:{self.name}" - - -@dataclass(frozen=True) -class DbtCombinedGraph: - """Combined downstream-only dbt and SQLBuild graph.""" - - nodes: frozenset[DbtCombinedGraphKey] - upstream_deps: dict[DbtCombinedGraphKey, tuple[DbtCombinedGraphKey, ...]] - downstream_deps: dict[DbtCombinedGraphKey, tuple[DbtCombinedGraphKey, ...]] - - @dataclass(frozen=True) class DbtInteropParsedArgs: """Declared `sqb dbt` flags parsed by argparse, before routing to buckets.""" diff --git a/src/sqlbuild/integrations/dbt/types.py b/src/sqlbuild/integrations/dbt/types.py index 9943d0c53f..c9558180ba 100644 --- a/src/sqlbuild/integrations/dbt/types.py +++ b/src/sqlbuild/integrations/dbt/types.py @@ -13,20 +13,6 @@ class DbtInvoker(Protocol): def __call__(self, *, argv: tuple[str, ...], cwd: Path | None) -> object: ... -class DbtCombinedGraphOwner(StrEnum): - """Owner namespace for a combined dbt/SQLBuild graph node.""" - - DBT = "dbt" - SQLBUILD = "sqb" - - -class DbtCombinedGraphResourceType(StrEnum): - """Resource type namespace for a combined dbt/SQLBuild graph node.""" - - MODEL = "model" - SOURCE = "source" - - class DbtSupportedResourceType(StrEnum): """dbt resource types SQLBuild currently handles with dedicated behavior.""" diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py index 4a8e449605..647476caad 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py @@ -6,22 +6,18 @@ import pytest from sqlbuild._native import ( + DbtCombinedGraph, assemble_project, build_compile_inputs, + build_dbt_combined_graph, build_dbt_manifest_index, discover_project_inputs, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt._helpers.graph.core import ( - build_dbt_combined_graph, - dbt_model_graph_key, - expand_combined_downstream, -) from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.models import ( DbtCliOptions, - DbtCombinedGraph, DbtCommandResult, DbtManifestIndex, ) @@ -33,7 +29,7 @@ build_external_sql_reference_resolver, build_sqlbuild_project_with_manifest, ) -from tests.unit.src.sqlbuild.integrations.dbt.helpers import graph_key_stable_ids +from tests.unit.src.sqlbuild.integrations.dbt.helpers import downstream_stable_ids pytestmark: pytest.MarkDecorator = pytest.mark.dbt @@ -94,11 +90,8 @@ def test_given_real_dbt_manifest_and_sqlbuild_project_when_building_graph_then_e graph: DbtCombinedGraph = build_dbt_combined_graph(manifest=manifest, project=project) assert ( - graph_key_stable_ids( - expand_combined_downstream( - key=dbt_model_graph_key(test_case.expected_downstream_from), - downstream=graph.downstream_deps, - ) + downstream_stable_ids( + graph=graph, stable_id=f"dbt:model:{test_case.expected_downstream_from}" ) == test_case.expected_downstream_keys ) diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index e8f31ae397..7d94e0304e 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -8,10 +8,14 @@ from typing import cast from sqlbuild._native import ( + DbtCombinedGraph, DbtEventStream, DbtEventStreamUpdate, assemble_project, + build_dbt_combined_graph, + build_dbt_manifest_index, extract_sql_references, + transitive_closure, ) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -34,22 +38,12 @@ ) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.integrations.dbt._helpers.cli.runner import build_dbt_ls_argv -from sqlbuild.integrations.dbt._helpers.graph.core import ( - dbt_model_graph_key, - dbt_source_graph_key, - sqlbuild_model_graph_key, -) from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.models import ( DbtCliConfigOverrides, DbtCliOptions, - DbtCombinedGraphKey, DbtCommandResult, ) -from sqlbuild.integrations.dbt.types import ( - DbtCombinedGraphOwner, - DbtCombinedGraphResourceType, -) from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig _GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() @@ -458,50 +452,37 @@ def build_compiled_project_with_model_specs( ) -def graph_edge_stable_ids( - graph_edges: dict[DbtCombinedGraphKey, tuple[DbtCombinedGraphKey, ...]], -) -> dict[str, tuple[str, ...]]: - """Render graph edges as stable IDs for assertions.""" +def downstream_stable_ids(*, graph: DbtCombinedGraph, stable_id: str) -> tuple[str, ...]: + """Render keys transitively downstream of one graph key as sorted stable IDs.""" - stable_edges: dict[str, tuple[str, ...]] = {} - for key, deps in graph_edges.items(): - stable_deps: list[str] = [] - for dep in deps: - stable_deps.append(dep.stable_id) - stable_edges[key.stable_id] = tuple(stable_deps) - return stable_edges + return tuple(sorted(transitive_closure(start=stable_id, edges=graph.downstream_deps))) -def graph_key_stable_ids(keys: frozenset[DbtCombinedGraphKey]) -> tuple[str, ...]: - """Render graph keys as sorted stable IDs for assertions.""" +def upstream_stable_ids(*, graph: DbtCombinedGraph, stable_id: str) -> tuple[str, ...]: + """Render keys transitively upstream of one graph key as sorted stable IDs.""" - return tuple(sorted(key.stable_id for key in keys)) + return tuple(sorted(transitive_closure(start=stable_id, edges=graph.upstream_deps))) -def graph_key_from_stable_id(stable_id: str) -> DbtCombinedGraphKey: - """Build a graph key from its stable string form.""" +def build_orders_dbt_graph() -> DbtCombinedGraph: + """Build a graph where SQLBuild `local_orders` reads dbt `model.analytics.orders`.""" - owner, resource_type, name = stable_id.split(":", maxsplit=2) - owner_enum: DbtCombinedGraphOwner = DbtCombinedGraphOwner(owner) - resource_type_enum: DbtCombinedGraphResourceType = DbtCombinedGraphResourceType(resource_type) - factories: dict[ - tuple[DbtCombinedGraphOwner, DbtCombinedGraphResourceType], - Callable[[str], DbtCombinedGraphKey], - ] = { - ( - DbtCombinedGraphOwner.DBT, - DbtCombinedGraphResourceType.SOURCE, - ): dbt_source_graph_key, - ( - DbtCombinedGraphOwner.DBT, - DbtCombinedGraphResourceType.MODEL, - ): dbt_model_graph_key, - ( - DbtCombinedGraphOwner.SQLBUILD, - DbtCombinedGraphResourceType.MODEL, - ): sqlbuild_model_graph_key, - } - return factories[(owner_enum, resource_type_enum)](name) + return build_dbt_combined_graph( + manifest=build_dbt_manifest_index( + raw_data=build_manifest_data( + nodes=( + build_manifest_model_node( + unique_id="model.analytics.orders", + package_name="analytics", + name="orders", + ), + ) + ) + ), + project=build_compiled_project_with_models( + {"local_orders": 'select * from __dbt_ref("orders")'} + ), + ) def consume_dbt_events(*events: dict[str, object]) -> DbtEventStreamUpdate | None: diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py index aef13370a8..1b00aa9140 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild._native import build_dbt_interop_plan +from sqlbuild._native import DbtCombinedGraph, build_dbt_interop_plan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs @@ -22,8 +22,6 @@ from sqlbuild.integrations.dbt.main.pipeline import plan as plan_module from sqlbuild.integrations.dbt.models import ( DbtCliOptions, - DbtCombinedGraph, - DbtCombinedGraphKey, DbtCommandExecutionResult, DbtInteropCompiledProject, DbtInteropExecutionRequest, @@ -36,17 +34,16 @@ DbtManifestIndex, DbtNodeExecutionResult, ) -from sqlbuild.integrations.dbt.types import ( - DbtCombinedGraphOwner, - DbtCombinedGraphResourceType, - DbtInteropCommand, -) +from sqlbuild.integrations.dbt.types import DbtInteropCommand from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtCompileFullRefreshPipelineTestCase, DbtExecutionSelectionStatusTestCase, DbtExecutionSummaryFooterTestCase, ) -from tests.unit.src.sqlbuild.integrations.dbt.helpers import CompileOnlyDbtRunner +from tests.unit.src.sqlbuild.integrations.dbt.helpers import ( + CompileOnlyDbtRunner, + build_orders_dbt_graph, +) @pytest.mark.parametrize( @@ -188,7 +185,7 @@ def test_given_ordinary_execution_when_compiling_then_never_uses_dbt_state( execute_module, "resolve_dbt_interop_plan", lambda **kwargs: DbtInteropPlanResolution( - graph=DbtCombinedGraph(nodes=frozenset(), upstream_deps={}, downstream_deps={}), + graph=build_orders_dbt_graph(), plan=plan, ), ) @@ -274,21 +271,8 @@ def execute_commands(**kwargs: object) -> DbtCommandExecutionResult: def test_given_actual_dbt_failure_when_resolving_blockers_then_blocks_downstream_sqlbuild_work( test_case: DbtExecutionSummaryFooterTestCase, ) -> None: - dbt_key: DbtCombinedGraphKey = DbtCombinedGraphKey( - owner=DbtCombinedGraphOwner.DBT, - resource_type=DbtCombinedGraphResourceType.MODEL, - name="model.analytics.orders", - ) - sqlbuild_key: DbtCombinedGraphKey = DbtCombinedGraphKey( - owner=DbtCombinedGraphOwner.SQLBUILD, - resource_type=DbtCombinedGraphResourceType.MODEL, - name="local_orders", - ) - graph: DbtCombinedGraph = DbtCombinedGraph( - nodes=frozenset((dbt_key, sqlbuild_key)), - upstream_deps={dbt_key: (), sqlbuild_key: (dbt_key,)}, - downstream_deps={dbt_key: (sqlbuild_key,), sqlbuild_key: ()}, - ) + graph: DbtCombinedGraph = build_orders_dbt_graph() + dbt_unique_id: str = "model.analytics.orders" result: tuple[str, ...] = build_failed_sqlbuild_model_names( graph=graph, @@ -299,7 +283,7 @@ def test_given_actual_dbt_failure_when_resolving_blockers_then_blocks_downstream ), node_results=( DbtNodeExecutionResult( - unique_id=dbt_key.name, + unique_id=dbt_unique_id, resource_type="model", node_name="orders", status=test_case.node_statuses[0], @@ -327,27 +311,14 @@ def test_given_actual_dbt_failure_when_resolving_blockers_then_blocks_downstream def test_given_failed_dbt_test_when_resolving_blockers_then_blocks_tested_model_downstream( test_case: DbtExecutionSummaryFooterTestCase, ) -> None: - dbt_key: DbtCombinedGraphKey = DbtCombinedGraphKey( - owner=DbtCombinedGraphOwner.DBT, - resource_type=DbtCombinedGraphResourceType.MODEL, - name="model.analytics.orders", - ) - sqlbuild_key: DbtCombinedGraphKey = DbtCombinedGraphKey( - owner=DbtCombinedGraphOwner.SQLBUILD, - resource_type=DbtCombinedGraphResourceType.MODEL, - name="local_orders", - ) - graph: DbtCombinedGraph = DbtCombinedGraph( - nodes=frozenset((dbt_key, sqlbuild_key)), - upstream_deps={dbt_key: (), sqlbuild_key: (dbt_key,)}, - downstream_deps={dbt_key: (sqlbuild_key,), sqlbuild_key: ()}, - ) + graph: DbtCombinedGraph = build_orders_dbt_graph() + dbt_unique_id: str = "model.analytics.orders" test_unique_id: str = "test.analytics.unique_orders.0123456789" manifest: DbtManifestIndex = DbtManifestIndex( models_by_unique_id={}, models_by_name={}, models_by_package_and_name={}, - validation_depends_on_nodes_by_unique_id={test_unique_id: (dbt_key.name,)}, + validation_depends_on_nodes_by_unique_id={test_unique_id: (dbt_unique_id,)}, ) result: tuple[str, ...] = build_failed_sqlbuild_model_names( diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py b/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py index 7d37df9880..a030759270 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py @@ -2,23 +2,17 @@ import pytest -from sqlbuild._native import build_dbt_manifest_index +from sqlbuild._native import DbtCombinedGraph, build_dbt_combined_graph, build_dbt_manifest_index from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.integrations.dbt._helpers.graph.core import ( - build_dbt_combined_graph, - expand_combined_downstream, - expand_combined_upstream, -) -from sqlbuild.integrations.dbt.models import DbtCombinedGraph, DbtManifestIndex +from sqlbuild.integrations.dbt.models import DbtManifestIndex from tests.unit.src.sqlbuild.integrations.dbt._test_types import DbtCombinedGraphTestCase from tests.unit.src.sqlbuild.integrations.dbt.helpers import ( build_compiled_project_with_models, build_manifest_data, build_manifest_model_node, build_manifest_source_node, - graph_edge_stable_ids, - graph_key_from_stable_id, - graph_key_stable_ids, + downstream_stable_ids, + upstream_stable_ids, ) @@ -124,22 +118,12 @@ def test_given_manifest_and_compiled_project_when_building_graph_then_returns_ex graph: DbtCombinedGraph = build_dbt_combined_graph(manifest=manifest, project=project) - assert graph_edge_stable_ids(graph.upstream_deps) == dict(test_case.expected_upstream_edges) + assert graph.upstream_deps == dict(test_case.expected_upstream_edges) assert ( - graph_key_stable_ids( - expand_combined_downstream( - key=graph_key_from_stable_id(test_case.expected_downstream_from), - downstream=graph.downstream_deps, - ) - ) + downstream_stable_ids(graph=graph, stable_id=test_case.expected_downstream_from) == test_case.expected_downstream_keys ) assert ( - graph_key_stable_ids( - expand_combined_upstream( - key=graph_key_from_stable_id(test_case.expected_upstream_from), - upstream=graph.upstream_deps, - ) - ) + upstream_stable_ids(graph=graph, stable_id=test_case.expected_upstream_from) == test_case.expected_upstream_keys ) diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py b/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py index bf0deca744..7af5b5b789 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py @@ -4,16 +4,14 @@ import pytest -from sqlbuild._native import build_dbt_manifest_index +from sqlbuild._native import DbtCombinedGraph, build_dbt_combined_graph, build_dbt_manifest_index from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.integrations.dbt._helpers.cli.runner import build_dbt_ls_argv -from sqlbuild.integrations.dbt._helpers.graph.core import build_dbt_combined_graph from sqlbuild.integrations.dbt._helpers.planning.orchestration import plan_dbt_interop_command from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError from sqlbuild.integrations.dbt.models import ( DbtCliOptions, - DbtCombinedGraph, DbtCommandResult, DbtInteropCommandArgs, DbtInteropPlan, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py b/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py index b564c604da..8e552420cc 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py @@ -2,16 +2,15 @@ import pytest -from sqlbuild._native import build_dbt_manifest_index -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.integrations.dbt._helpers.graph.core import build_dbt_combined_graph -from sqlbuild.integrations.dbt._helpers.selection.core import resolve_dbt_interop_sqlbuild_selection -from sqlbuild.integrations.dbt.exceptions import DbtInteropArgumentError -from sqlbuild.integrations.dbt.models import ( +from sqlbuild._native import ( DbtCombinedGraph, - DbtInteropSelectionResult, - DbtManifestIndex, + build_dbt_combined_graph, + build_dbt_manifest_index, + resolve_dbt_interop_sqlbuild_selection, ) +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.integrations.dbt.exceptions import DbtInteropArgumentError +from sqlbuild.integrations.dbt.models import DbtInteropSelectionResult, DbtManifestIndex from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtSelectionErrorTestCase, DbtSelectionTestCase, From aa884918116b0b28e063aa2d74695b83a8c4aae6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:11:25 +0100 Subject: [PATCH 21/24] refactor: build dbt argv, route sqb dbt arguments and resolve dbt config natively --- .../src/dbt_cli/_helpers/argv.rs | 33 ++ .../sqlbuild-core/src/dbt_cli/_helpers/mod.rs | 2 + .../src/dbt_cli/_helpers/routing.rs | 187 ++++++++ crates/sqlbuild-core/src/dbt_cli/constants.rs | 38 ++ crates/sqlbuild-core/src/dbt_cli/errors.rs | 9 + .../src/dbt_cli/main/build_dbt_argv.rs | 60 +++ .../src/dbt_cli/main/choose_dbt_config.rs | 31 ++ crates/sqlbuild-core/src/dbt_cli/main/mod.rs | 4 + .../dbt_cli/main/resolve_dbt_executable.rs | 13 + .../dbt_cli/main/route_dbt_interop_args.rs | 38 ++ crates/sqlbuild-core/src/dbt_cli/mod.rs | 10 + crates/sqlbuild-core/src/dbt_cli/models.rs | 114 +++++ crates/sqlbuild-core/src/dbt_cli/tests.rs | 6 + .../src/dbt_cli/tests/helpers.rs | 48 ++ .../src/dbt_cli/tests/test_dbt_cli.rs | 279 ++++++++++++ .../src/dbt_cli/tests/test_types.rs | 22 + crates/sqlbuild-core/src/lib.rs | 1 + .../src/bindings/_helpers/dbt_interop/cli.rs | 410 ++++++++++++++++++ .../src/bindings/_helpers/dbt_interop/mod.rs | 1 + .../_helpers/dbt_interop/registration.rs | 3 +- src/sqlbuild/_native.pyi | 42 ++ .../integrations/dbt/_helpers/cli/args.py | 183 -------- .../integrations/dbt/_helpers/cli/runner.py | 161 ------- .../integrations/dbt/_helpers/config/core.py | 88 ---- .../dbt/_helpers/pipeline/execute.py | 10 +- .../dbt/_helpers/pipeline/interop_prologue.py | 4 +- .../dbt/_helpers/planning/runtime.py | 2 +- .../integrations/dbt/classes/dbt_runner.py | 14 +- src/sqlbuild/integrations/dbt/constants.py | 3 - .../dbt/main/cli/_build_command_argv.py | 18 - .../dbt/main/cli/_build_compile_argv.py | 12 - .../dbt/main/cli/_build_debug_argv.py | 14 - .../dbt/main/cli/_build_deps_argv.py | 10 - .../dbt/main/cli/_build_ls_argv.py | 25 -- .../dbt/main/cli/_parse_ls_json_lines.py | 10 - .../dbt/main/cli/_resolve_executable.py | 9 - .../dbt/main/cli/_route_interop_args.py | 13 - .../dbt/main/config/_resolve_config.py | 26 -- .../integrations/dbt/main/pipeline/debug.py | 3 +- src/sqlbuild/integrations/dbt/models.py | 1 - .../src/sqlbuild/integrations/dbt/helpers.py | 2 +- .../sqlbuild/integrations/dbt/test_args.py | 2 +- .../sqlbuild/integrations/dbt/test_config.py | 2 +- .../dbt/test_plan_orchestration.py | 8 +- .../sqlbuild/integrations/dbt/test_runner.py | 2 +- 45 files changed, 1379 insertions(+), 594 deletions(-) create mode 100644 crates/sqlbuild-core/src/dbt_cli/_helpers/argv.rs create mode 100644 crates/sqlbuild-core/src/dbt_cli/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/dbt_cli/_helpers/routing.rs create mode 100644 crates/sqlbuild-core/src/dbt_cli/constants.rs create mode 100644 crates/sqlbuild-core/src/dbt_cli/errors.rs create mode 100644 crates/sqlbuild-core/src/dbt_cli/main/build_dbt_argv.rs create mode 100644 crates/sqlbuild-core/src/dbt_cli/main/choose_dbt_config.rs create mode 100644 crates/sqlbuild-core/src/dbt_cli/main/mod.rs create mode 100644 crates/sqlbuild-core/src/dbt_cli/main/resolve_dbt_executable.rs create mode 100644 crates/sqlbuild-core/src/dbt_cli/main/route_dbt_interop_args.rs create mode 100644 crates/sqlbuild-core/src/dbt_cli/mod.rs create mode 100644 crates/sqlbuild-core/src/dbt_cli/models.rs create mode 100644 crates/sqlbuild-core/src/dbt_cli/tests.rs create mode 100644 crates/sqlbuild-core/src/dbt_cli/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/dbt_cli/tests/test_dbt_cli.rs create mode 100644 crates/sqlbuild-core/src/dbt_cli/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs delete mode 100644 src/sqlbuild/integrations/dbt/_helpers/cli/args.py delete mode 100644 src/sqlbuild/integrations/dbt/_helpers/cli/runner.py delete mode 100644 src/sqlbuild/integrations/dbt/_helpers/config/core.py delete mode 100644 src/sqlbuild/integrations/dbt/main/cli/_build_command_argv.py delete mode 100644 src/sqlbuild/integrations/dbt/main/cli/_build_compile_argv.py delete mode 100644 src/sqlbuild/integrations/dbt/main/cli/_build_debug_argv.py delete mode 100644 src/sqlbuild/integrations/dbt/main/cli/_build_deps_argv.py delete mode 100644 src/sqlbuild/integrations/dbt/main/cli/_build_ls_argv.py delete mode 100644 src/sqlbuild/integrations/dbt/main/cli/_parse_ls_json_lines.py delete mode 100644 src/sqlbuild/integrations/dbt/main/cli/_resolve_executable.py delete mode 100644 src/sqlbuild/integrations/dbt/main/cli/_route_interop_args.py delete mode 100644 src/sqlbuild/integrations/dbt/main/config/_resolve_config.py diff --git a/crates/sqlbuild-core/src/dbt_cli/_helpers/argv.rs b/crates/sqlbuild-core/src/dbt_cli/_helpers/argv.rs new file mode 100644 index 0000000000..a7cfe80edd --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/_helpers/argv.rs @@ -0,0 +1,33 @@ +//! dbt options rendered as command-line flags. + +use crate::dbt_cli::models::DbtCliOptionValues; + +fn with_value(mut argv: Vec, flag: &str, value: Option<&String>) -> Vec { + if let Some(value) = value { + argv.push(flag.to_owned()); + argv.push(value.clone()); + } + argv +} + +/// `argv` followed by every set common option, in dbt's documented order. +pub(crate) fn with_common_options(argv: Vec, options: &DbtCliOptionValues) -> Vec { + let argv: Vec = with_value(argv, "--project-dir", options.project_dir.as_ref()); + let argv: Vec = with_value(argv, "--profiles-dir", options.profiles_dir.as_ref()); + let argv: Vec = with_value(argv, "--target", options.target.as_ref()); + let argv: Vec = with_value(argv, "--target-path", options.target_path.as_ref()); + let argv: Vec = with_value(argv, "--vars", options.vars.as_ref()); + let mut argv: Vec = with_value(argv, "--state", options.state.as_ref()); + if options.defer { + argv.push("--defer".to_owned()); + } + argv +} + +/// `argv` followed by the options `dbt deps` accepts. +pub(crate) fn with_deps_options(argv: Vec, options: &DbtCliOptionValues) -> Vec { + let argv: Vec = with_value(argv, "--project-dir", options.project_dir.as_ref()); + let argv: Vec = with_value(argv, "--profiles-dir", options.profiles_dir.as_ref()); + let argv: Vec = with_value(argv, "--target", options.target.as_ref()); + with_value(argv, "--vars", options.vars.as_ref()) +} diff --git a/crates/sqlbuild-core/src/dbt_cli/_helpers/mod.rs b/crates/sqlbuild-core/src/dbt_cli/_helpers/mod.rs new file mode 100644 index 0000000000..885d322b2a --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod argv; +pub(crate) mod routing; diff --git a/crates/sqlbuild-core/src/dbt_cli/_helpers/routing.rs b/crates/sqlbuild-core/src/dbt_cli/_helpers/routing.rs new file mode 100644 index 0000000000..63a2cf8f1c --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/_helpers/routing.rs @@ -0,0 +1,187 @@ +//! `sqb dbt` flag validation and the dbt and SQLBuild argument buckets. + +use crate::dbt_cli::constants::{ + EVENT_TIME_CURSOR_CONFLICT_CODE, EVENT_TIME_CURSOR_CONFLICT_HELP, EVENT_TIME_PAIR_CODE, + EXECUTION_SQLBUILD_FLAGS, PLAN_SQLBUILD_FLAGS, SQLBUILD_FLAG_CODE, +}; +use crate::dbt_cli::errors::DbtCliError; +use crate::dbt_cli::models::{DbtInteropCommandKind, DbtParsedFlags, DbtRoutedArgs}; + +fn maps_to_sqlbuild(command: DbtInteropCommandKind) -> bool { + command != DbtInteropCommandKind::Debug +} + +fn pushed(mut args: Vec, values: &[&str]) -> Vec { + args.extend(values.iter().map(|value| (*value).to_owned())); + args +} + +fn allowed_sqlbuild_flags(command: DbtInteropCommandKind) -> &'static [&'static str] { + match command { + DbtInteropCommandKind::Run | DbtInteropCommandKind::Build => &EXECUTION_SQLBUILD_FLAGS, + DbtInteropCommandKind::Plan => &PLAN_SQLBUILD_FLAGS, + DbtInteropCommandKind::Debug => &[], + } +} + +/// Reject an event-time start without an end, or the reverse. +pub(crate) fn check_event_time_pair(flags: &DbtParsedFlags) -> Result<(), DbtCliError> { + if flags.event_time_start.is_none() == flags.event_time_end.is_none() { + return Ok(()); + } + Err(DbtCliError { + message: "--event-time-start and --event-time-end must be provided together".to_owned(), + code: Some(EVENT_TIME_PAIR_CODE), + help: None, + }) +} + +/// Reject SQLBuild-only flags the command does not accept. +pub(crate) fn check_sqlbuild_flags( + command: DbtInteropCommandKind, + flags: &DbtParsedFlags, +) -> Result<(), DbtCliError> { + let allowed: &[&str] = allowed_sqlbuild_flags(command); + let present: [(&str, bool); 7] = [ + ("--start-cursor-ts", flags.start_cursor_ts.is_some()), + ("--end-cursor-ts", flags.end_cursor_ts.is_some()), + ("--start-cursor-int", flags.start_cursor_int.is_some()), + ("--end-cursor-int", flags.end_cursor_int.is_some()), + ("--defer-to", flags.defer_to.is_some()), + ("--fail-fast", flags.fail_fast), + ("--hard-copy", flags.hard_copy), + ]; + match present + .into_iter() + .find(|(flag, present)| *present && !allowed.contains(flag)) + { + Some((flag, _)) => Err(DbtCliError { + message: format!( + "{flag} is not a valid SQLBuild option for sqb dbt {}", + command.name() + ), + code: Some(SQLBUILD_FLAG_CODE), + help: None, + }), + None => Ok(()), + } +} + +/// Reject dbt event-time flags combined with SQLBuild timestamp cursors. +pub(crate) fn check_event_time_cursors( + command: DbtInteropCommandKind, + flags: &DbtParsedFlags, +) -> Result<(), DbtCliError> { + if flags.event_time_start.is_none() || !maps_to_sqlbuild(command) { + return Ok(()); + } + if flags.start_cursor_ts.is_none() && flags.end_cursor_ts.is_none() { + return Ok(()); + } + Err(DbtCliError { + message: "--event-time-start/end conflict with --start-cursor-ts/end-cursor-ts".to_owned(), + code: Some(EVENT_TIME_CURSOR_CONFLICT_CODE), + help: Some(EVENT_TIME_CURSOR_CONFLICT_HELP), + }) +} + +/// Selection flags, then flags both tools receive, in routing order. +pub(crate) fn shared_args( + command: DbtInteropCommandKind, + flags: &DbtParsedFlags, + routed: DbtRoutedArgs, +) -> DbtRoutedArgs { + let DbtRoutedArgs { + mut dbt_args, + mut sqlbuild_args, + } = routed; + if !flags.select.is_empty() { + dbt_args.push("--select".to_owned()); + dbt_args.extend(flags.select.iter().cloned()); + } + if !flags.exclude.is_empty() { + dbt_args.push("--exclude".to_owned()); + dbt_args.extend(flags.exclude.iter().cloned()); + } + if let Some(vars) = &flags.vars { + dbt_args = pushed(dbt_args, &["--vars", vars]); + sqlbuild_args = pushed(sqlbuild_args, &["--vars", vars]); + } + if let Some(threads) = &flags.threads { + dbt_args = pushed(dbt_args, &["--threads", threads]); + sqlbuild_args = pushed(sqlbuild_args, &["--concurrency", threads]); + } + let to_sqlbuild: bool = maps_to_sqlbuild(command); + if flags.full_refresh { + dbt_args = pushed(dbt_args, &["--full-refresh"]); + if to_sqlbuild { + sqlbuild_args = pushed(sqlbuild_args, &["--full-refresh"]); + } + } + for (value, dbt_flag, sqlbuild_flag) in [ + ( + &flags.event_time_start, + "--event-time-start", + "--start-cursor-ts", + ), + (&flags.event_time_end, "--event-time-end", "--end-cursor-ts"), + ] { + if let Some(value) = value { + dbt_args = pushed(dbt_args, &[dbt_flag, value]); + if to_sqlbuild { + sqlbuild_args = pushed(sqlbuild_args, &[sqlbuild_flag, value]); + } + } + } + DbtRoutedArgs { + dbt_args, + sqlbuild_args, + } +} + +/// dbt configuration flags, then SQLBuild-only flags. +pub(crate) fn tool_args(flags: &DbtParsedFlags, routed: DbtRoutedArgs) -> DbtRoutedArgs { + let DbtRoutedArgs { + mut dbt_args, + mut sqlbuild_args, + } = routed; + for (flag, value) in [ + ("--project-dir", &flags.project_dir), + ("--profiles-dir", &flags.profiles_dir), + ("--profile", &flags.profile), + ("--target", &flags.target), + ("--target-path", &flags.target_path), + ("--state", &flags.state), + ("--indirect-selection", &flags.indirect_selection), + ] { + if let Some(value) = value { + dbt_args = pushed(dbt_args, &[flag, value]); + } + } + if flags.defer { + dbt_args = pushed(dbt_args, &["--defer"]); + } + for (flag, value) in [ + ("--start-cursor-ts", &flags.start_cursor_ts), + ("--end-cursor-ts", &flags.end_cursor_ts), + ("--start-cursor-int", &flags.start_cursor_int), + ("--end-cursor-int", &flags.end_cursor_int), + ("--defer-to", &flags.defer_to), + ] { + if let Some(value) = value { + sqlbuild_args = pushed(sqlbuild_args, &[flag, value]); + } + } + for (flag, enabled) in [ + ("--fail-fast", flags.fail_fast), + ("--hard-copy", flags.hard_copy), + ] { + if enabled { + sqlbuild_args = pushed(sqlbuild_args, &[flag]); + } + } + DbtRoutedArgs { + dbt_args, + sqlbuild_args, + } +} diff --git a/crates/sqlbuild-core/src/dbt_cli/constants.rs b/crates/sqlbuild-core/src/dbt_cli/constants.rs new file mode 100644 index 0000000000..09bd04e0cb --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/constants.rs @@ -0,0 +1,38 @@ +//! dbt executable defaults, flags and routing error codes. + +/// The dbt executable when no override is set. +pub const DEFAULT_DBT_EXECUTABLE: &str = "dbt"; +/// SQLBuild cursor flags `sqb dbt` forwards to SQLBuild. +pub const CURSOR_FLAGS: [&str; 4] = [ + "--start-cursor-ts", + "--end-cursor-ts", + "--start-cursor-int", + "--end-cursor-int", +]; +/// SQLBuild-only flags `sqb dbt run` and `sqb dbt build` accept. +pub const EXECUTION_SQLBUILD_FLAGS: [&str; 7] = [ + "--start-cursor-ts", + "--end-cursor-ts", + "--start-cursor-int", + "--end-cursor-int", + "--defer-to", + "--fail-fast", + "--verbose", +]; +/// SQLBuild-only flags `sqb dbt plan` accepts. +pub const PLAN_SQLBUILD_FLAGS: [&str; 5] = [ + "--start-cursor-ts", + "--end-cursor-ts", + "--start-cursor-int", + "--end-cursor-int", + "--defer-to", +]; +pub const UNSUPPORTED_COMMAND_CODE: &str = "C236"; +pub const EVENT_TIME_PAIR_CODE: &str = "C233"; +pub const EVENT_TIME_CURSOR_CONFLICT_CODE: &str = "C234"; +pub const SQLBUILD_FLAG_CODE: &str = "C232"; +pub const EVENT_TIME_CURSOR_CONFLICT_HELP: &str = + "Use dbt event-time flags or SQLBuild timestamp cursor overrides, not both."; +/// Help for a missing dbt project directory. +pub const MISSING_PROJECT_DIR_HELP: &str = "to point SQLBuild at the dbt project, set this in sqlbuild_project.toml:\n [dbt]\n project_dir = \"path/to/dbt_project\"\n = help: or pass `--project-dir` to `sqb dbt`"; +pub const MISSING_PROJECT_DIR_MESSAGE: &str = "dbt project directory is not configured"; diff --git a/crates/sqlbuild-core/src/dbt_cli/errors.rs b/crates/sqlbuild-core/src/dbt_cli/errors.rs new file mode 100644 index 0000000000..cd688ef22b --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/errors.rs @@ -0,0 +1,9 @@ +//! Invalid `sqb dbt` arguments and configuration. + +/// A rejected argument combination or missing setting, with its diagnostic code and help. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtCliError { + pub message: String, + pub code: Option<&'static str>, + pub help: Option<&'static str>, +} diff --git a/crates/sqlbuild-core/src/dbt_cli/main/build_dbt_argv.rs b/crates/sqlbuild-core/src/dbt_cli/main/build_dbt_argv.rs new file mode 100644 index 0000000000..8b7526da39 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/main/build_dbt_argv.rs @@ -0,0 +1,60 @@ +//! Build the argv of one dbt command. + +use crate::dbt_cli::_helpers::argv::{with_common_options, with_deps_options}; +use crate::dbt_cli::models::{DbtArgvCommand, DbtCliOptionValues}; + +/// `dbt_executable `. +#[must_use] +pub fn build_dbt_argv( + dbt_executable: &str, + options: &DbtCliOptionValues, + command: DbtArgvCommand, +) -> Vec { + let start = |names: &[&str]| -> Vec { + std::iter::once(dbt_executable) + .chain(names.iter().copied()) + .map(str::to_owned) + .collect() + }; + match command { + DbtArgvCommand::Compile { full_refresh } => { + let mut argv: Vec = with_common_options(start(&["compile"]), options); + if full_refresh { + argv.push("--full-refresh".to_owned()); + } + argv + } + DbtArgvCommand::Deps => with_deps_options(start(&["deps"]), options), + DbtArgvCommand::Debug { args } => { + let mut argv: Vec = with_common_options(start(&["debug"]), options); + argv.extend(args); + argv + } + DbtArgvCommand::Other { command, args } => { + let mut argv: Vec = with_common_options(start(&[&command]), options); + argv.extend(args); + argv + } + DbtArgvCommand::Ls { + select, + exclude, + resource_types, + } => { + let mut argv: Vec = + with_common_options(start(&["ls", "--output", "json"]), options); + if !select.is_empty() { + argv.push("--select".to_owned()); + argv.extend(select); + } + if !exclude.is_empty() { + argv.push("--exclude".to_owned()); + argv.extend(exclude); + } + for resource_type in resource_types { + argv.push("--resource-type".to_owned()); + argv.push(resource_type); + } + argv + } + } +} diff --git a/crates/sqlbuild-core/src/dbt_cli/main/choose_dbt_config.rs b/crates/sqlbuild-core/src/dbt_cli/main/choose_dbt_config.rs new file mode 100644 index 0000000000..c9a002bb66 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/main/choose_dbt_config.rs @@ -0,0 +1,31 @@ +//! Choose dbt settings from CLI overrides, local config and project config. + +use crate::dbt_cli::constants::{MISSING_PROJECT_DIR_HELP, MISSING_PROJECT_DIR_MESSAGE}; +use crate::dbt_cli::errors::DbtCliError; +use crate::dbt_cli::models::{DbtConfigChoice, DbtConfigSources}; + +/// CLI overrides win over project config; `target` also consults local config first. +pub fn choose_dbt_config( + sources: DbtConfigSources, + require_project_dir: bool, +) -> Result { + let project_dir: Option = sources.override_project_dir.or(sources.config_project_dir); + if require_project_dir && project_dir.is_none() { + return Err(DbtCliError { + message: MISSING_PROJECT_DIR_MESSAGE.to_owned(), + code: None, + help: Some(MISSING_PROJECT_DIR_HELP), + }); + } + Ok(DbtConfigChoice { + project_dir, + profiles_dir: sources + .override_profiles_dir + .or(sources.config_profiles_dir), + target_path: sources.override_target_path.or(sources.config_target_path), + target: sources + .override_target + .or(sources.local_target) + .or(sources.config_target), + }) +} diff --git a/crates/sqlbuild-core/src/dbt_cli/main/mod.rs b/crates/sqlbuild-core/src/dbt_cli/main/mod.rs new file mode 100644 index 0000000000..bb436c444f --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/main/mod.rs @@ -0,0 +1,4 @@ +pub mod build_dbt_argv; +pub mod choose_dbt_config; +pub mod resolve_dbt_executable; +pub mod route_dbt_interop_args; diff --git a/crates/sqlbuild-core/src/dbt_cli/main/resolve_dbt_executable.rs b/crates/sqlbuild-core/src/dbt_cli/main/resolve_dbt_executable.rs new file mode 100644 index 0000000000..89ee819cf5 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/main/resolve_dbt_executable.rs @@ -0,0 +1,13 @@ +//! Choose the dbt executable. + +use crate::dbt_cli::constants::DEFAULT_DBT_EXECUTABLE; +use crate::text::main::python_strip::python_strip; + +/// The stripped override when it is not blank, else `dbt`. +#[must_use] +pub fn resolve_dbt_executable(override_value: Option<&str>) -> String { + match override_value.map(python_strip) { + Some(value) if !value.is_empty() => value.to_owned(), + _ => DEFAULT_DBT_EXECUTABLE.to_owned(), + } +} diff --git a/crates/sqlbuild-core/src/dbt_cli/main/route_dbt_interop_args.rs b/crates/sqlbuild-core/src/dbt_cli/main/route_dbt_interop_args.rs new file mode 100644 index 0000000000..8bed0f1f0a --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/main/route_dbt_interop_args.rs @@ -0,0 +1,38 @@ +//! Split parsed `sqb dbt` flags into dbt and SQLBuild arguments. + +use crate::dbt_cli::_helpers::routing::{ + check_event_time_cursors, check_event_time_pair, check_sqlbuild_flags, shared_args, tool_args, +}; +use crate::dbt_cli::constants::UNSUPPORTED_COMMAND_CODE; +use crate::dbt_cli::errors::DbtCliError; +use crate::dbt_cli::models::{DbtInteropCommandKind, DbtParsedFlags, DbtRoutedArgs}; + +/// The command and its routed argument buckets, dbt passthrough arguments last. +pub fn route_dbt_interop_args( + command: &str, + flags: &DbtParsedFlags, +) -> Result<(DbtInteropCommandKind, DbtRoutedArgs), DbtCliError> { + let kind: DbtInteropCommandKind = match command { + "plan" => DbtInteropCommandKind::Plan, + "run" => DbtInteropCommandKind::Run, + "build" => DbtInteropCommandKind::Build, + "debug" => DbtInteropCommandKind::Debug, + _ => { + return Err(DbtCliError { + message: format!("Unsupported sqb dbt command: {command}"), + code: Some(UNSUPPORTED_COMMAND_CODE), + help: None, + }); + } + }; + check_event_time_pair(flags)?; + check_sqlbuild_flags(kind, flags)?; + let routed: DbtRoutedArgs = + tool_args(flags, shared_args(kind, flags, DbtRoutedArgs::default())); + check_event_time_cursors(kind, flags)?; + let mut routed: DbtRoutedArgs = routed; + routed + .dbt_args + .extend(flags.dbt_passthrough.iter().cloned()); + Ok((kind, routed)) +} diff --git a/crates/sqlbuild-core/src/dbt_cli/mod.rs b/crates/sqlbuild-core/src/dbt_cli/mod.rs new file mode 100644 index 0000000000..d154cd3077 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/mod.rs @@ -0,0 +1,10 @@ +//! dbt command lines: argv building, executable choice, `sqb dbt` argument routing and config choice. + +mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/dbt_cli/models.rs b/crates/sqlbuild-core/src/dbt_cli/models.rs new file mode 100644 index 0000000000..9c68d57504 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/models.rs @@ -0,0 +1,114 @@ +//! dbt options, argv requests, parsed `sqb dbt` flags and config sources. + +/// dbt options shared by every dbt command, paths already rendered as text. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtCliOptionValues { + pub project_dir: Option, + pub profiles_dir: Option, + pub target: Option, + pub target_path: Option, + pub vars: Option, + pub state: Option, + pub defer: bool, +} + +/// The dbt command an argv is built for and its command-specific arguments. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DbtArgvCommand { + Compile { + full_refresh: bool, + }, + Deps, + Debug { + args: Vec, + }, + Other { + command: String, + args: Vec, + }, + Ls { + select: Vec, + exclude: Vec, + resource_types: Vec, + }, +} + +/// The `sqb dbt` command a routed argument set belongs to. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DbtInteropCommandKind { + Plan, + Run, + Build, + Debug, +} + +impl DbtInteropCommandKind { + /// The command's name on the command line. + #[must_use] + pub fn name(self) -> &'static str { + match self { + Self::Plan => "plan", + Self::Run => "run", + Self::Build => "build", + Self::Debug => "debug", + } + } +} + +/// Declared `sqb dbt` flags as parsed, before routing. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtParsedFlags { + pub select: Vec, + pub exclude: Vec, + pub vars: Option, + pub threads: Option, + pub full_refresh: bool, + pub event_time_start: Option, + pub event_time_end: Option, + pub project_dir: Option, + pub profiles_dir: Option, + pub profile: Option, + pub target: Option, + pub target_path: Option, + pub state: Option, + pub indirect_selection: Option, + pub defer: bool, + pub start_cursor_ts: Option, + pub end_cursor_ts: Option, + pub start_cursor_int: Option, + pub end_cursor_int: Option, + pub defer_to: Option, + pub fail_fast: bool, + pub hard_copy: bool, + pub dbt_passthrough: Vec, +} + +/// Parsed flags split into the dbt and SQLBuild argument buckets. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtRoutedArgs { + pub dbt_args: Vec, + pub sqlbuild_args: Vec, +} + +/// One dbt setting from a CLI override and project config, plus a local override for `target`. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtConfigSources { + pub override_project_dir: Option, + pub config_project_dir: Option, + pub override_profiles_dir: Option, + pub config_profiles_dir: Option, + pub override_target_path: Option, + pub config_target_path: Option, + pub override_target: Option, + pub local_target: Option, + pub config_target: Option, +} + +/// The raw dbt setting values that apply, paths still unresolved. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtConfigChoice { + pub project_dir: Option, + pub profiles_dir: Option, + pub target_path: Option, + pub target: Option, +} diff --git a/crates/sqlbuild-core/src/dbt_cli/tests.rs b/crates/sqlbuild-core/src/dbt_cli/tests.rs new file mode 100644 index 0000000000..0a5b3d14a0 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_dbt_cli.rs"] +mod dbt_cli; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/dbt_cli/tests/helpers.rs b/crates/sqlbuild-core/src/dbt_cli/tests/helpers.rs new file mode 100644 index 0000000000..6065c20ea5 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/tests/helpers.rs @@ -0,0 +1,48 @@ +use crate::dbt_cli::errors::DbtCliError; +use crate::dbt_cli::models::{DbtCliOptionValues, DbtInteropCommandKind, DbtRoutedArgs}; + +pub(super) fn text(value: &str) -> Option { + Some(value.to_owned()) +} + +pub(super) fn texts(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// Options with every common setting set. +pub(super) fn full_options() -> DbtCliOptionValues { + DbtCliOptionValues { + project_dir: text("/repo/dbt"), + profiles_dir: text("/repo/profiles"), + target: text("prod"), + target_path: text("/repo/target"), + vars: text("{}"), + state: text("/repo/state"), + defer: true, + } +} + +/// The routed buckets as text, or empty buckets on failure. +pub(super) fn buckets( + routed: &Result<(DbtInteropCommandKind, DbtRoutedArgs), DbtCliError>, +) -> (Vec<&str>, Vec<&str>) { + routed.as_ref().map_or_else( + |_| (Vec::new(), Vec::new()), + |(_, args)| { + ( + args.dbt_args.iter().map(String::as_str).collect(), + args.sqlbuild_args.iter().map(String::as_str).collect(), + ) + }, + ) +} + +/// The failure message and code, when routing failed. +pub(super) fn failure( + routed: &Result<(DbtInteropCommandKind, DbtRoutedArgs), DbtCliError>, +) -> Option<(&str, &str)> { + routed + .as_ref() + .err() + .map(|error| (error.message.as_str(), error.code.unwrap_or_default())) +} diff --git a/crates/sqlbuild-core/src/dbt_cli/tests/test_dbt_cli.rs b/crates/sqlbuild-core/src/dbt_cli/tests/test_dbt_cli.rs new file mode 100644 index 0000000000..a5be010e1d --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/tests/test_dbt_cli.rs @@ -0,0 +1,279 @@ +use crate::dbt_cli::main::build_dbt_argv::build_dbt_argv; +use crate::dbt_cli::main::choose_dbt_config::choose_dbt_config; +use crate::dbt_cli::main::resolve_dbt_executable::resolve_dbt_executable; +use crate::dbt_cli::main::route_dbt_interop_args::route_dbt_interop_args; +use crate::dbt_cli::models::{ + DbtArgvCommand, DbtCliOptionValues, DbtConfigChoice, DbtConfigSources, DbtParsedFlags, +}; +use crate::dbt_cli::tests::helpers::{buckets, failure, full_options, text, texts}; +use crate::dbt_cli::tests::test_types::{ArgvTestCase, ConfigTestCase, RouteTestCase}; + +#[test] +fn given_options_when_building_argv_then_orders_flags_like_dbt() { + let test_cases = [ + ArgvTestCase { + description: "compile with every option and full refresh", + argv: build_dbt_argv( + "dbt", + &full_options(), + DbtArgvCommand::Compile { full_refresh: true }, + ), + expected_argv: vec![ + "dbt", + "compile", + "--project-dir", + "/repo/dbt", + "--profiles-dir", + "/repo/profiles", + "--target", + "prod", + "--target-path", + "/repo/target", + "--vars", + "{}", + "--state", + "/repo/state", + "--defer", + "--full-refresh", + ], + }, + ArgvTestCase { + description: "deps only passes project, profiles, target and vars", + argv: build_dbt_argv("dbt", &full_options(), DbtArgvCommand::Deps), + expected_argv: vec![ + "dbt", + "deps", + "--project-dir", + "/repo/dbt", + "--profiles-dir", + "/repo/profiles", + "--target", + "prod", + "--vars", + "{}", + ], + }, + ArgvTestCase { + description: "ls with selectors and resource types", + argv: build_dbt_argv( + "bin/dbt", + &DbtCliOptionValues::default(), + DbtArgvCommand::Ls { + select: texts(&["a", "b+"]), + exclude: texts(&["c"]), + resource_types: texts(&["model", "seed"]), + }, + ), + expected_argv: vec![ + "bin/dbt", + "ls", + "--output", + "json", + "--select", + "a", + "b+", + "--exclude", + "c", + "--resource-type", + "model", + "--resource-type", + "seed", + ], + }, + ArgvTestCase { + description: "another command keeps its arguments last", + argv: build_dbt_argv( + "dbt", + &DbtCliOptionValues { + target: text("dev"), + ..DbtCliOptionValues::default() + }, + DbtArgvCommand::Other { + command: "build".to_owned(), + args: texts(&["--threads", "4"]), + }, + ), + expected_argv: vec!["dbt", "build", "--target", "dev", "--threads", "4"], + }, + ArgvTestCase { + description: "executable override is stripped and blank falls back", + argv: vec![ + resolve_dbt_executable(Some(" /opt/dbt \n")), + resolve_dbt_executable(Some(" \t")), + resolve_dbt_executable(None), + ], + expected_argv: vec!["/opt/dbt", "dbt", "dbt"], + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_case.argv, test_case.expected_argv, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_parsed_flags_when_routing_then_splits_buckets_or_rejects() { + let flags: DbtParsedFlags = DbtParsedFlags { + select: texts(&["orders+"]), + vars: text("{a: 1}"), + threads: text("4"), + full_refresh: true, + event_time_start: text("2026-01-01"), + event_time_end: text("2026-01-02"), + target: text("prod"), + defer: true, + defer_to: text("prod"), + fail_fast: true, + dbt_passthrough: texts(&["--warn-error"]), + ..DbtParsedFlags::default() + }; + let test_cases = [ + RouteTestCase { + description: "run routes shared, dbt and SQLBuild flags in order", + routed: route_dbt_interop_args("run", &flags), + expected_dbt_args: vec![ + "--select", + "orders+", + "--vars", + "{a: 1}", + "--threads", + "4", + "--full-refresh", + "--event-time-start", + "2026-01-01", + "--event-time-end", + "2026-01-02", + "--target", + "prod", + "--defer", + "--warn-error", + ], + expected_sqlbuild_args: vec![ + "--vars", + "{a: 1}", + "--concurrency", + "4", + "--full-refresh", + "--start-cursor-ts", + "2026-01-01", + "--end-cursor-ts", + "2026-01-02", + "--defer-to", + "prod", + "--fail-fast", + ], + expected_error: None, + }, + RouteTestCase { + description: "plan rejects --fail-fast", + routed: route_dbt_interop_args("plan", &flags), + expected_dbt_args: vec![], + expected_sqlbuild_args: vec![], + expected_error: Some(( + "--fail-fast is not a valid SQLBuild option for sqb dbt plan", + "C232", + )), + }, + RouteTestCase { + description: "unpaired event time", + routed: route_dbt_interop_args( + "debug", + &DbtParsedFlags { + event_time_end: text("x"), + ..DbtParsedFlags::default() + }, + ), + expected_dbt_args: vec![], + expected_sqlbuild_args: vec![], + expected_error: Some(( + "--event-time-start and --event-time-end must be provided together", + "C233", + )), + }, + RouteTestCase { + description: "event time with timestamp cursors", + routed: route_dbt_interop_args( + "build", + &DbtParsedFlags { + event_time_start: text("a"), + event_time_end: text("b"), + end_cursor_ts: text("c"), + ..DbtParsedFlags::default() + }, + ), + expected_dbt_args: vec![], + expected_sqlbuild_args: vec![], + expected_error: Some(( + "--event-time-start/end conflict with --start-cursor-ts/end-cursor-ts", + "C234", + )), + }, + RouteTestCase { + description: "unknown command", + routed: route_dbt_interop_args("seed", &DbtParsedFlags::default()), + expected_dbt_args: vec![], + expected_sqlbuild_args: vec![], + expected_error: Some(("Unsupported sqb dbt command: seed", "C236")), + }, + ]; + + for test_case in test_cases { + assert_eq!( + buckets(&test_case.routed), + ( + test_case.expected_dbt_args, + test_case.expected_sqlbuild_args + ), + "{}", + test_case.description + ); + assert_eq!( + failure(&test_case.routed), + test_case.expected_error, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_config_sources_when_choosing_then_prefers_overrides() { + let sources: DbtConfigSources = DbtConfigSources { + override_project_dir: text("cli"), + config_project_dir: text("config"), + config_profiles_dir: text("profiles"), + local_target: text("local"), + config_target: text("config"), + ..DbtConfigSources::default() + }; + let test_cases = [ + ConfigTestCase { + description: "overrides, then local target, then project config", + choice: choose_dbt_config(sources.clone(), true), + expected_choice: Ok(DbtConfigChoice { + project_dir: text("cli"), + profiles_dir: text("profiles"), + target_path: None, + target: text("local"), + }), + }, + ConfigTestCase { + description: "a required project directory must be set", + choice: choose_dbt_config(DbtConfigSources::default(), true), + expected_choice: Err("dbt project directory is not configured"), + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_case.choice.map_err(|error| error.message), + test_case.expected_choice.map_err(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/dbt_cli/tests/test_types.rs b/crates/sqlbuild-core/src/dbt_cli/tests/test_types.rs new file mode 100644 index 0000000000..037be62406 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_cli/tests/test_types.rs @@ -0,0 +1,22 @@ +use crate::dbt_cli::errors::DbtCliError; +use crate::dbt_cli::models::{DbtConfigChoice, DbtInteropCommandKind, DbtRoutedArgs}; + +pub(super) struct ArgvTestCase { + pub(super) description: &'static str, + pub(super) argv: Vec, + pub(super) expected_argv: Vec<&'static str>, +} + +pub(super) struct RouteTestCase { + pub(super) description: &'static str, + pub(super) routed: Result<(DbtInteropCommandKind, DbtRoutedArgs), DbtCliError>, + pub(super) expected_dbt_args: Vec<&'static str>, + pub(super) expected_sqlbuild_args: Vec<&'static str>, + pub(super) expected_error: Option<(&'static str, &'static str)>, +} + +pub(super) struct ConfigTestCase { + pub(super) description: &'static str, + pub(super) choice: Result, + pub(super) expected_choice: Result, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 83e662bc9d..277f12461b 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -11,6 +11,7 @@ pub mod constants; pub mod contract_command; pub mod cost_command; pub mod cost_report; +pub mod dbt_cli; pub mod dbt_command; pub mod dbt_events; pub mod dbt_graph; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs new file mode 100644 index 0000000000..65c4d9a2f5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs @@ -0,0 +1,410 @@ +//! dbt argv building, executable choice, `dbt ls` output, `sqb dbt` argument routing and config. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use serde_json::{Map, Value}; +use sqlbuild_core::dbt_cli::errors::DbtCliError; +use sqlbuild_core::dbt_cli::main::build_dbt_argv::build_dbt_argv; +use sqlbuild_core::dbt_cli::main::choose_dbt_config::choose_dbt_config; +use sqlbuild_core::dbt_cli::main::resolve_dbt_executable::resolve_dbt_executable as executable; +use sqlbuild_core::dbt_cli::main::route_dbt_interop_args::route_dbt_interop_args as route; +use sqlbuild_core::dbt_cli::models::{ + DbtArgvCommand, DbtCliOptionValues, DbtConfigChoice, DbtConfigSources, DbtInteropCommandKind, + DbtParsedFlags, DbtRoutedArgs, +}; +use sqlbuild_core::text::main::python_splitlines::python_splitlines; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; +const DBT_TYPES: &str = "sqlbuild.integrations.dbt.types"; +const DBT_EXCEPTIONS: &str = "sqlbuild.integrations.dbt.exceptions"; +const DBT_EXECUTABLE_ENV_VAR: &str = "DBT_EXECUTABLE"; +const COMMAND_ARGV_OPTIONS: (&[&str], &[&str]) = + (&["dbt_executable", "command", "options"], &["args"]); +const LS_ARGV_OPTIONS: (&[&str], &[&str]) = ( + &["dbt_executable", "options"], + &["select", "exclude", "resource_types"], +); +const CONFIG_OPTIONS: (&[&str], &[&str]) = ( + &["project_root", "config", "overrides", "require_project_dir"], + &["local_config"], +); + +fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.extract()?)) +} + +fn attribute_text(object: &Bound<'_, PyAny>, name: &str) -> PyResult> { + optional_text(&object.getattr(name)?) +} + +fn texts(values: &Bound<'_, PyAny>) -> PyResult> { + values.try_iter()?.map(|value| value?.extract()).collect() +} + +fn option_values(options: &Bound<'_, PyAny>) -> PyResult { + Ok(DbtCliOptionValues { + project_dir: attribute_text(options, "project_dir")?, + profiles_dir: attribute_text(options, "profiles_dir")?, + target: attribute_text(options, "target")?, + target_path: attribute_text(options, "target_path")?, + vars: attribute_text(options, "vars")?, + state: attribute_text(options, "state")?, + defer: options.getattr("defer")?.is_truthy()?, + }) +} + +fn argv<'py>( + py: Python<'py>, + (dbt_executable, options): (&str, &Bound<'py, PyAny>), + command: DbtArgvCommand, +) -> PyResult> { + PyTuple::new( + py, + build_dbt_argv(dbt_executable, &option_values(options)?, command), + ) +} + +fn sequence(values: &Bound<'_, PyAny>) -> PyResult> { + if values.is_none() { + return Ok(Vec::new()); + } + texts(values) +} + +/// The dbt executable, honouring a non-blank `DBT_EXECUTABLE` override. +#[pyfunction] +fn resolve_dbt_executable() -> String { + let value: Option = + std::env::var_os(DBT_EXECUTABLE_ENV_VAR).map(|value| value.to_string_lossy().into_owned()); + executable(value.as_deref()) +} + +/// argv for `dbt compile`. +#[pyfunction] +#[pyo3(signature = (*, dbt_executable, options, full_refresh = false))] +fn build_dbt_compile_argv<'py>( + py: Python<'py>, + dbt_executable: &str, + options: &Bound<'py, PyAny>, + full_refresh: bool, +) -> PyResult> { + argv( + py, + (dbt_executable, options), + DbtArgvCommand::Compile { full_refresh }, + ) +} + +/// argv for `dbt deps`. +#[pyfunction] +#[pyo3(signature = (*, dbt_executable, options))] +fn build_dbt_deps_argv<'py>( + py: Python<'py>, + dbt_executable: &str, + options: &Bound<'py, PyAny>, +) -> PyResult> { + argv(py, (dbt_executable, options), DbtArgvCommand::Deps) +} + +/// argv for `dbt debug`. +#[pyfunction] +#[pyo3(signature = (*, dbt_executable, options, args = None))] +fn build_dbt_debug_argv<'py>( + py: Python<'py>, + dbt_executable: &str, + options: &Bound<'py, PyAny>, + args: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let args: Vec = args.map_or_else(|| Ok(Vec::new()), sequence)?; + argv( + py, + (dbt_executable, options), + DbtArgvCommand::Debug { args }, + ) +} + +/// argv for one executable dbt command with the common options. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_dbt_command_argv<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "build_dbt_command_argv", options, COMMAND_ARGV_OPTIONS)?; + let command: DbtArgvCommand = DbtArgvCommand::Other { + command: arguments.value("command")?.extract()?, + args: sequence(&arguments.value("args")?)?, + }; + let dbt_executable: String = arguments.value("dbt_executable")?.extract()?; + argv(py, (&dbt_executable, &arguments.value("options")?), command) +} + +/// argv for `dbt ls --output json`. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_dbt_ls_argv<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "build_dbt_ls_argv", options, LS_ARGV_OPTIONS)?; + let command: DbtArgvCommand = DbtArgvCommand::Ls { + select: sequence(&arguments.value("select")?)?, + exclude: sequence(&arguments.value("exclude")?)?, + resource_types: sequence(&arguments.value("resource_types")?)?, + }; + let dbt_executable: String = arguments.value("dbt_executable")?.extract()?; + argv(py, (&dbt_executable, &arguments.value("options")?), command) +} + +fn json_text(payload: &Map, key: &str) -> Option { + payload.get(key).and_then(Value::as_str).map(str::to_owned) +} + +fn json_fqn(payload: &Map) -> Vec { + let Some(parts) = payload.get("fqn").and_then(Value::as_array) else { + return Vec::new(); + }; + let mut names: Vec = Vec::new(); + for part in parts { + match part.as_str() { + Some(name) if !name.is_empty() => names.push(name.to_owned()), + _ => return Vec::new(), + } + } + names +} + +fn ls_node<'py>( + py: Python<'py>, + node_class: &Bound<'py, PyAny>, + payload: &Map, +) -> PyResult>> { + let Some(unique_id) = json_text(payload, "unique_id").filter(|id| !id.is_empty()) else { + return Ok(None); + }; + let optional = |key: &str| -> Bound<'py, PyAny> { + json_text(payload, key).map_or_else( + || py.None().into_bound(py), + |value| PyString::new(py, &value).into_any(), + ) + }; + let node: Bound<'py, PyAny> = call_with( + node_class, + &[ + ("unique_id", PyString::new(py, &unique_id).into_any()), + ("resource_type", optional("resource_type")), + ("package_name", optional("package_name")), + ("name", optional("name")), + ("fqn", PyTuple::new(py, json_fqn(payload))?.into_any()), + ("original_file_path", optional("original_file_path")), + ], + )?; + Ok(Some(node)) +} + +/// Nodes from `dbt ls` JSON lines, skipping log lines, non-objects and nodes without an ID. +#[pyfunction] +#[pyo3(signature = (*, stdout))] +fn parse_dbt_ls_json_lines<'py>(py: Python<'py>, stdout: &str) -> PyResult> { + let node_class: Bound<'py, PyAny> = imported(py, DBT_MODELS, "DbtLsNode")?; + let mut nodes: Vec> = Vec::new(); + for line in python_splitlines(stdout) { + let stripped: &str = python_strip(line); + if !stripped.starts_with('{') { + continue; + } + let Ok(Value::Object(payload)) = serde_json::from_str::(stripped) else { + continue; + }; + if let Some(node) = ls_node(py, &node_class, &payload)? { + nodes.push(node); + } + } + PyTuple::new(py, nodes) +} + +fn raised(py: Python<'_>, class_name: &str, error: DbtCliError) -> PyErr { + let created = || -> PyResult { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + if let Some(code) = error.code { + keywords.set_item("code", code)?; + } + if let Some(help) = error.help { + keywords.set_item("help", help)?; + } + let instance: Bound<'_, PyAny> = + imported(py, DBT_EXCEPTIONS, class_name)?.call((error.message,), Some(&keywords))?; + Ok(PyErr::from_value(instance)) + }; + created().unwrap_or_else(|failure| failure) +} + +fn parsed_flags(parsed: &Bound<'_, PyAny>) -> PyResult { + let value = |name: &str| attribute_text(parsed, name); + let flag = |name: &str| -> PyResult { parsed.getattr(name)?.is_truthy() }; + Ok(DbtParsedFlags { + select: texts(&parsed.getattr("select")?)?, + exclude: texts(&parsed.getattr("exclude")?)?, + vars: value("vars")?, + threads: value("threads")?, + full_refresh: flag("full_refresh")?, + event_time_start: value("event_time_start")?, + event_time_end: value("event_time_end")?, + project_dir: value("project_dir")?, + profiles_dir: value("profiles_dir")?, + profile: value("profile")?, + target: value("target")?, + target_path: value("target_path")?, + state: value("state")?, + indirect_selection: value("indirect_selection")?, + defer: flag("defer")?, + start_cursor_ts: value("start_cursor_ts")?, + end_cursor_ts: value("end_cursor_ts")?, + start_cursor_int: value("start_cursor_int")?, + end_cursor_int: value("end_cursor_int")?, + defer_to: value("defer_to")?, + fail_fast: flag("fail_fast")?, + hard_copy: flag("hard_copy")?, + dbt_passthrough: texts(&parsed.getattr("dbt_passthrough")?)?, + }) +} + +/// Split parsed `sqb dbt ` flags into dbt and SQLBuild buckets. +#[pyfunction] +#[pyo3(signature = (*, command, parsed))] +fn route_dbt_interop_args<'py>( + py: Python<'py>, + command: &Bound<'py, PyAny>, + parsed: &Bound<'py, PyAny>, +) -> PyResult> { + let command: String = command.str()?.extract()?; + let flags: DbtParsedFlags = parsed_flags(parsed)?; + let (kind, routed): (DbtInteropCommandKind, DbtRoutedArgs) = + route(&command, &flags).map_err(|error| raised(py, "DbtInteropArgumentError", error))?; + let command: Bound<'py, PyAny> = + imported(py, DBT_TYPES, "DbtInteropCommand")?.call1((kind.name(),))?; + call_with( + &imported(py, DBT_MODELS, "DbtInteropRoutedArgs")?, + &[ + ("command", command), + ("select", PyTuple::new(py, flags.select)?.into_any()), + ("exclude", PyTuple::new(py, flags.exclude)?.into_any()), + ("dbt_args", PyTuple::new(py, routed.dbt_args)?.into_any()), + ( + "sqlbuild_args", + PyTuple::new(py, routed.sqlbuild_args)?.into_any(), + ), + ], + ) +} + +/// `Path(raw).expanduser()` resolved, relative paths against the resolved project root. +fn resolved_path<'py>( + project_root: &Bound<'py, PyAny>, + raw_value: Option, +) -> PyResult> { + let py: Python<'py> = project_root.py(); + let Some(raw_value) = raw_value else { + return Ok(py.None().into_bound(py)); + }; + let root: Bound<'py, PyAny> = project_root + .call_method0("expanduser")? + .call_method0("resolve")?; + let path: Bound<'py, PyAny> = imported(py, "pathlib", "Path")? + .call1((raw_value,))? + .call_method0("expanduser")?; + if path.call_method0("is_absolute")?.is_truthy()? { + return path.call_method0("resolve"); + } + root.div(&path)?.call_method0("resolve") +} + +fn config_sources( + config: &Bound<'_, PyAny>, + overrides: &Bound<'_, PyAny>, + local_config: &Bound<'_, PyAny>, +) -> PyResult { + let local_target: Option = if local_config.is_none() { + None + } else { + attribute_text(local_config, "target")? + }; + Ok(DbtConfigSources { + override_project_dir: attribute_text(overrides, "project_dir")?, + config_project_dir: attribute_text(config, "project_dir")?, + override_profiles_dir: attribute_text(overrides, "profiles_dir")?, + config_profiles_dir: attribute_text(config, "profiles_dir")?, + override_target_path: attribute_text(overrides, "target_path")?, + config_target_path: attribute_text(config, "target_path")?, + override_target: attribute_text(overrides, "target")?, + local_target, + config_target: attribute_text(config, "target")?, + }) +} + +/// Resolve dbt config from CLI overrides, local config and project config. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_dbt_config<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "resolve_dbt_config", options, CONFIG_OPTIONS)?; + let project_root: Bound<'py, PyAny> = arguments.value("project_root")?; + let sources: DbtConfigSources = config_sources( + &arguments.value("config")?, + &arguments.value("overrides")?, + &arguments.value("local_config")?, + )?; + let choice: DbtConfigChoice = + choose_dbt_config(sources, arguments.flag("require_project_dir")?) + .map_err(|error| raised(py, "DbtInteropConfigError", error))?; + let target: Bound<'py, PyAny> = choice.target.map_or_else( + || py.None().into_bound(py), + |target| PyString::new(py, &target).into_any(), + ); + call_with( + &imported(py, DBT_MODELS, "ResolvedDbtConfig")?, + &[ + ( + "project_dir", + resolved_path(&project_root, choice.project_dir)?, + ), + ( + "profiles_dir", + resolved_path(&project_root, choice.profiles_dir)?, + ), + ("target", target), + ( + "target_path", + resolved_path(&project_root, choice.target_path)?, + ), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_dbt_executable, module)?)?; + module.add_function(wrap_pyfunction!(build_dbt_compile_argv, module)?)?; + module.add_function(wrap_pyfunction!(build_dbt_deps_argv, module)?)?; + module.add_function(wrap_pyfunction!(build_dbt_debug_argv, module)?)?; + module.add_function(wrap_pyfunction!(build_dbt_command_argv, module)?)?; + module.add_function(wrap_pyfunction!(build_dbt_ls_argv, module)?)?; + module.add_function(wrap_pyfunction!(parse_dbt_ls_json_lines, module)?)?; + module.add_function(wrap_pyfunction!(route_dbt_interop_args, module)?)?; + module.add_function(wrap_pyfunction!(resolve_dbt_config, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs index e05c3f9b53..4046369c14 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod cli; pub(crate) mod graph; pub(crate) mod manifest; pub(crate) mod plan; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs index 4a673eec06..439de2d4c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::dbt_interop::{graph, manifest, plan, profile}; +use crate::bindings::_helpers::dbt_interop::{cli, graph, manifest, plan, profile}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + cli::register(module)?; graph::register(module)?; manifest::register(module)?; plan::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b8accc4be4..87942ca75b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -261,13 +261,18 @@ from sqlbuild.executor.python_nodes.models import ( from sqlbuild.executor.scenario.models import ScenarioRunResult, ScenarioSnapshotCaptureRunResult from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.integrations.dbt.models import ( + DbtCliConfigOverrides, + DbtCliOptions, + DbtInteropParsedArgs, DbtInteropPlan, + DbtInteropRoutedArgs, DbtInteropSelectionResult, DbtLsNode, DbtManifestIndex, DbtManifestModel, DbtNodeExecutionResult, NormalizedDbtProfileConnection, + ResolvedDbtConfig, ResolvedDbtProfileOutput, ) from sqlbuild.integrations.dbt.types import DbtInteropCommand @@ -284,9 +289,11 @@ from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.runtime.observability.models import LifecycleEvent from sqlbuild.spec.contracts.models import ( + DbtConfig, DefaultsConfig, ExecutionLimitsConfig, LocalConfig, + LocalDbtConfig, ProjectConfig, ScenarioConfig, SchemaAuditInstance, @@ -1483,6 +1490,41 @@ def resolve_dbt_interop_sqlbuild_selection( dbt_anchor_unique_ids_by_term: Mapping[str, Sequence[str]] | None = None, ) -> DbtInteropSelectionResult: ... def dbt_fqn_selector_term(*, fqn: Sequence[str], fallback: str) -> str: ... +def resolve_dbt_executable() -> str: ... +def build_dbt_compile_argv( + *, dbt_executable: str, options: DbtCliOptions, full_refresh: bool = False +) -> tuple[str, ...]: ... +def build_dbt_deps_argv(*, dbt_executable: str, options: DbtCliOptions) -> tuple[str, ...]: ... +def build_dbt_debug_argv( + *, dbt_executable: str, options: DbtCliOptions, args: Sequence[str] | None = None +) -> tuple[str, ...]: ... +def build_dbt_command_argv( + *, + dbt_executable: str, + command: str, + options: DbtCliOptions, + args: Sequence[str] | None = None, +) -> tuple[str, ...]: ... +def build_dbt_ls_argv( + *, + dbt_executable: str, + options: DbtCliOptions, + select: Sequence[str] | None = None, + exclude: Sequence[str] | None = None, + resource_types: Sequence[str] | None = None, +) -> tuple[str, ...]: ... +def parse_dbt_ls_json_lines(*, stdout: str) -> tuple[DbtLsNode, ...]: ... +def route_dbt_interop_args( + *, command: DbtInteropCommand | str, parsed: DbtInteropParsedArgs +) -> DbtInteropRoutedArgs: ... +def resolve_dbt_config( + *, + project_root: Path, + config: DbtConfig, + overrides: DbtCliConfigOverrides, + require_project_dir: bool, + local_config: LocalDbtConfig | None = None, +) -> ResolvedDbtConfig: ... def resolve_dbt_reference_relation( *, manifest: DbtManifestIndex | None, diff --git a/src/sqlbuild/integrations/dbt/_helpers/cli/args.py b/src/sqlbuild/integrations/dbt/_helpers/cli/args.py deleted file mode 100644 index ca9ced88a7..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/cli/args.py +++ /dev/null @@ -1,183 +0,0 @@ -"""Argument routing helpers for `sqb dbt` commands.""" - -from __future__ import annotations - -from sqlbuild.integrations.dbt.exceptions import DbtInteropArgumentError -from sqlbuild.integrations.dbt.models import DbtInteropParsedArgs, DbtInteropRoutedArgs -from sqlbuild.integrations.dbt.types import DbtInteropCommand - - -def route_dbt_interop_args( - *, command: DbtInteropCommand | str, parsed: DbtInteropParsedArgs -) -> DbtInteropRoutedArgs: - """Split parsed `sqb dbt ` flags into dbt and SQLBuild buckets.""" - - try: - normalized_command: DbtInteropCommand = DbtInteropCommand(command) - except ValueError as exc: - raise DbtInteropArgumentError( - f"Unsupported sqb dbt command: {command}", - code="C236", - ) from exc - _validate_event_time_pair(parsed) - _validate_sqlbuild_flags_allowed(command=normalized_command, parsed=parsed) - dbt_args: list[str] = [] - sqlbuild_args: list[str] = [] - dbt_args = _route_selection(parsed=parsed, dbt_args=dbt_args) - dbt_args, sqlbuild_args = _route_shared( - command=normalized_command, parsed=parsed, dbt_args=dbt_args, sqlbuild_args=sqlbuild_args - ) - dbt_args = _route_dbt_config(parsed=parsed, dbt_args=dbt_args) - sqlbuild_args = _route_sqlbuild_only(parsed=parsed, sqlbuild_args=sqlbuild_args) - _validate_event_time_cursor_conflict(command=normalized_command, parsed=parsed) - dbt_args.extend(parsed.dbt_passthrough) - return DbtInteropRoutedArgs( - command=normalized_command, - select=tuple(parsed.select), - exclude=tuple(parsed.exclude), - dbt_args=tuple(dbt_args), - sqlbuild_args=tuple(sqlbuild_args), - ) - - -def _route_selection(*, parsed: DbtInteropParsedArgs, dbt_args: list[str]) -> list[str]: - if parsed.select: - dbt_args.extend(("--select", *parsed.select)) - if parsed.exclude: - dbt_args.extend(("--exclude", *parsed.exclude)) - return dbt_args - - -def _route_shared( - *, - command: DbtInteropCommand, - parsed: DbtInteropParsedArgs, - dbt_args: list[str], - sqlbuild_args: list[str], -) -> tuple[list[str], list[str]]: - if parsed.vars is not None: - dbt_args.extend(("--vars", parsed.vars)) - sqlbuild_args.extend(("--vars", parsed.vars)) - if parsed.threads is not None: - dbt_args.extend(("--threads", parsed.threads)) - sqlbuild_args.extend(("--concurrency", parsed.threads)) - if parsed.full_refresh: - dbt_args.append("--full-refresh") - if _map_full_refresh_to_sqlbuild(command): - sqlbuild_args.append("--full-refresh") - if parsed.event_time_start is not None: - dbt_args.extend(("--event-time-start", parsed.event_time_start)) - if _map_event_time_to_sqlbuild(command): - sqlbuild_args.extend(("--start-cursor-ts", parsed.event_time_start)) - if parsed.event_time_end is not None: - dbt_args.extend(("--event-time-end", parsed.event_time_end)) - if _map_event_time_to_sqlbuild(command): - sqlbuild_args.extend(("--end-cursor-ts", parsed.event_time_end)) - return dbt_args, sqlbuild_args - - -def _route_dbt_config(*, parsed: DbtInteropParsedArgs, dbt_args: list[str]) -> list[str]: - for flag, value in ( - ("--project-dir", parsed.project_dir), - ("--profiles-dir", parsed.profiles_dir), - ("--profile", parsed.profile), - ("--target", parsed.target), - ("--target-path", parsed.target_path), - ("--state", parsed.state), - ("--indirect-selection", parsed.indirect_selection), - ): - if value is not None: - dbt_args.extend((flag, value)) - if parsed.defer: - dbt_args.append("--defer") - return dbt_args - - -def _route_sqlbuild_only(*, parsed: DbtInteropParsedArgs, sqlbuild_args: list[str]) -> list[str]: - for flag, value in ( - ("--start-cursor-ts", parsed.start_cursor_ts), - ("--end-cursor-ts", parsed.end_cursor_ts), - ("--start-cursor-int", parsed.start_cursor_int), - ("--end-cursor-int", parsed.end_cursor_int), - ("--defer-to", parsed.defer_to), - ): - if value is not None: - sqlbuild_args.extend((flag, value)) - for flag, enabled in ( - ("--fail-fast", parsed.fail_fast), - ("--hard-copy", parsed.hard_copy), - ): - if enabled: - sqlbuild_args.append(flag) - return sqlbuild_args - - -def _validate_event_time_pair(parsed: DbtInteropParsedArgs) -> None: - if (parsed.event_time_start is None) == (parsed.event_time_end is None): - return - raise DbtInteropArgumentError( - "--event-time-start and --event-time-end must be provided together", - code="C233", - ) - - -def _validate_event_time_cursor_conflict( - *, command: DbtInteropCommand, parsed: DbtInteropParsedArgs -) -> None: - if parsed.event_time_start is None or not _map_event_time_to_sqlbuild(command): - return - if parsed.start_cursor_ts is None and parsed.end_cursor_ts is None: - return - raise DbtInteropArgumentError( - "--event-time-start/end conflict with --start-cursor-ts/end-cursor-ts", - code="C234", - help="Use dbt event-time flags or SQLBuild timestamp cursor overrides, not both.", - ) - - -def _validate_sqlbuild_flags_allowed( - *, command: DbtInteropCommand, parsed: DbtInteropParsedArgs -) -> None: - allowed: frozenset[str] = _allowed_sqlbuild_flags(command) - for flag, present in ( - ("--start-cursor-ts", parsed.start_cursor_ts is not None), - ("--end-cursor-ts", parsed.end_cursor_ts is not None), - ("--start-cursor-int", parsed.start_cursor_int is not None), - ("--end-cursor-int", parsed.end_cursor_int is not None), - ("--defer-to", parsed.defer_to is not None), - ("--fail-fast", parsed.fail_fast), - ("--hard-copy", parsed.hard_copy), - ): - if present and flag not in allowed: - raise DbtInteropArgumentError( - f"{flag} is not a valid SQLBuild option for sqb dbt {command.value}", - code="C232", - ) - - -def _allowed_sqlbuild_flags(command: DbtInteropCommand) -> frozenset[str]: - cursor_flags: tuple[str, ...] = ( - "--start-cursor-ts", - "--end-cursor-ts", - "--start-cursor-int", - "--end-cursor-int", - ) - common_execution_flags: tuple[str, ...] = ( - *cursor_flags, - "--defer-to", - "--fail-fast", - "--verbose", - ) - if command in {DbtInteropCommand.RUN, DbtInteropCommand.BUILD}: - return frozenset(common_execution_flags) - if command == DbtInteropCommand.PLAN: - return frozenset((*cursor_flags, "--defer-to")) - return frozenset() - - -def _map_full_refresh_to_sqlbuild(command: DbtInteropCommand) -> bool: - return command in {DbtInteropCommand.PLAN, DbtInteropCommand.RUN, DbtInteropCommand.BUILD} - - -def _map_event_time_to_sqlbuild(command: DbtInteropCommand) -> bool: - return command in {DbtInteropCommand.PLAN, DbtInteropCommand.RUN, DbtInteropCommand.BUILD} diff --git a/src/sqlbuild/integrations/dbt/_helpers/cli/runner.py b/src/sqlbuild/integrations/dbt/_helpers/cli/runner.py deleted file mode 100644 index 3004359728..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/cli/runner.py +++ /dev/null @@ -1,161 +0,0 @@ -"""dbt CLI runner and `dbt ls` parser.""" - -from __future__ import annotations - -import json -import os -from collections.abc import Sequence - -from sqlbuild.integrations.dbt.constants import DBT_EXECUTABLE_ENV_VAR, DEFAULT_DBT_EXECUTABLE -from sqlbuild.integrations.dbt.models import ( - DbtCliOptions, - DbtLsNode, -) - - -def resolve_dbt_executable() -> str: - """Return the dbt executable, honoring the DBT_EXECUTABLE override.""" - - override: str | None = os.environ.get(DBT_EXECUTABLE_ENV_VAR) - if override is not None and override.strip(): - return override.strip() - return DEFAULT_DBT_EXECUTABLE - - -def build_dbt_compile_argv( - *, dbt_executable: str, options: DbtCliOptions, full_refresh: bool = False -) -> tuple[str, ...]: - """Build argv for dbt compile.""" - - argv: tuple[str, ...] = _append_common_options( - argv=(dbt_executable, "compile"), options=options - ) - if full_refresh: - argv = (*argv, "--full-refresh") - return argv - - -def build_dbt_deps_argv(*, dbt_executable: str, options: DbtCliOptions) -> tuple[str, ...]: - """Build argv for dbt deps.""" - - argv: tuple[str, ...] = (dbt_executable, "deps") - if options.project_dir is not None: - argv = (*argv, "--project-dir", str(options.project_dir)) - if options.profiles_dir is not None: - argv = (*argv, "--profiles-dir", str(options.profiles_dir)) - if options.target is not None: - argv = (*argv, "--target", options.target) - if options.vars is not None: - argv = (*argv, "--vars", options.vars) - return argv - - -def build_dbt_debug_argv( - *, dbt_executable: str, options: DbtCliOptions, args: Sequence[str] = () -) -> tuple[str, ...]: - """Build argv for dbt debug.""" - - return (*_append_common_options(argv=(dbt_executable, "debug"), options=options), *args) - - -def build_dbt_command_argv( - *, - dbt_executable: str, - command: str, - options: DbtCliOptions, - args: Sequence[str] = (), -) -> tuple[str, ...]: - """Build argv for an executable dbt command with resolved common options.""" - - return (*_append_common_options(argv=(dbt_executable, command), options=options), *args) - - -def build_dbt_ls_argv( - *, - dbt_executable: str, - options: DbtCliOptions, - select: Sequence[str] = (), - exclude: Sequence[str] = (), - resource_types: Sequence[str] = (), -) -> tuple[str, ...]: - """Build argv for dbt ls with JSON output.""" - - argv: tuple[str, ...] = _append_common_options( - argv=(dbt_executable, "ls", "--output", "json"), options=options - ) - value: str - if select: - argv = (*argv, "--select", *select) - if exclude: - argv = (*argv, "--exclude", *exclude) - for value in resource_types: - argv = (*argv, "--resource-type", value) - return argv - - -def parse_dbt_ls_json_lines(*, stdout: str) -> tuple[DbtLsNode, ...]: - """Parse dbt ls JSON-lines output, ignoring non-JSON log lines.""" - - nodes: list[DbtLsNode] = [] - line: str - for line in stdout.splitlines(): - stripped: str = line.strip() - if not stripped or not stripped.startswith("{"): - continue - try: - payload: object = json.loads(stripped) - except json.JSONDecodeError: - continue - if not isinstance(payload, dict): - continue - unique_id: object | None = payload.get("unique_id") - if not isinstance(unique_id, str) or not unique_id: - continue - nodes.append( - DbtLsNode( - unique_id=unique_id, - resource_type=_optional_str(payload.get("resource_type")), - package_name=_optional_str(payload.get("package_name")), - name=_optional_str(payload.get("name")), - fqn=_parse_fqn(payload.get("fqn")), - original_file_path=_optional_str(payload.get("original_file_path")), - payload=dict(payload), - ) - ) - return tuple(nodes) - - -def _append_common_options(*, argv: tuple[str, ...], options: DbtCliOptions) -> tuple[str, ...]: - if options.project_dir is not None: - argv = (*argv, "--project-dir", str(options.project_dir)) - if options.profiles_dir is not None: - argv = (*argv, "--profiles-dir", str(options.profiles_dir)) - if options.target is not None: - argv = (*argv, "--target", options.target) - if options.target_path is not None: - argv = (*argv, "--target-path", str(options.target_path)) - if options.vars is not None: - argv = (*argv, "--vars", options.vars) - if options.state is not None: - argv = (*argv, "--state", str(options.state)) - if options.defer: - argv = (*argv, "--defer") - return argv - - -def _optional_str(value: object | None) -> str | None: - if isinstance(value, str): - return value - return None - - -def _parse_fqn(value: object | None) -> tuple[str, ...]: - if not isinstance(value, list): - return () - parts: list[str] = [] - item: object - for item in value: - if not isinstance(item, str) or not item: - return () - parts.append(item) - return tuple(parts) diff --git a/src/sqlbuild/integrations/dbt/_helpers/config/core.py b/src/sqlbuild/integrations/dbt/_helpers/config/core.py deleted file mode 100644 index 0b7499d74c..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/config/core.py +++ /dev/null @@ -1,88 +0,0 @@ -"""dbt configuration resolution helpers.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery.constants import ( - DBT_PROJECT_DIR_KEY, - DBT_SECTION, - PROJECT_CONFIG_FILENAME, -) -from sqlbuild.errors.setting_help.main.join_helps import join_helps -from sqlbuild.errors.setting_help.main.setting_help import setting_help -from sqlbuild.integrations.dbt.exceptions import DbtInteropConfigError -from sqlbuild.integrations.dbt.models import DbtCliConfigOverrides, ResolvedDbtConfig -from sqlbuild.spec.contracts.models import DbtConfig, LocalDbtConfig - - -def resolve_dbt_config( - *, - project_root: Path, - config: DbtConfig, - overrides: DbtCliConfigOverrides, - require_project_dir: bool, - local_config: LocalDbtConfig | None = None, -) -> ResolvedDbtConfig: - """Resolve dbt config from CLI overrides and project config.""" - - raw_project_dir: str | None = ( - overrides.project_dir if overrides.project_dir is not None else config.project_dir - ) - project_dir: Path | None = _resolve_optional_path( - project_root=project_root, - raw_value=raw_project_dir, - ) - if require_project_dir and project_dir is None: - raise DbtInteropConfigError( - "dbt project directory is not configured", - help=join_helps( - setting_help( - purpose="to point SQLBuild at the dbt project", - file_name=PROJECT_CONFIG_FILENAME, - section=DBT_SECTION, - key=DBT_PROJECT_DIR_KEY, - value="path/to/dbt_project", - ), - "or pass `--project-dir` to `sqb dbt`", - ), - ) - - raw_profiles_dir: str | None = ( - overrides.profiles_dir if overrides.profiles_dir is not None else config.profiles_dir - ) - profiles_dir: Path | None = _resolve_optional_path( - project_root=project_root, - raw_value=raw_profiles_dir, - ) - raw_target_path: str | None = ( - overrides.target_path if overrides.target_path is not None else config.target_path - ) - target_path: Path | None = _resolve_optional_path( - project_root=project_root, - raw_value=raw_target_path, - ) - target: str | None = ( - overrides.target - if overrides.target is not None - else local_config.target - if local_config is not None and local_config.target is not None - else config.target - ) - - return ResolvedDbtConfig( - project_dir=project_dir, - profiles_dir=profiles_dir, - target=target, - target_path=target_path, - ) - - -def _resolve_optional_path(*, project_root: Path, raw_value: str | None) -> Path | None: - if raw_value is None: - return None - resolved_project_root: Path = project_root.expanduser().resolve() - path: Path = Path(raw_value).expanduser() - if path.is_absolute(): - return path.resolve() - return (resolved_project_root / path).resolve() diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py index 0aa494a0a4..30bd9b185f 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py @@ -8,7 +8,13 @@ from pathlib import Path from typing import TextIO, cast -from sqlbuild._native import DbtCombinedGraph, format_summary_footer, render_dbt_node_result +from sqlbuild._native import ( + DbtCombinedGraph, + build_dbt_command_argv, + format_summary_footer, + parse_dbt_ls_json_lines, + render_dbt_node_result, +) from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.constants import ( DBT_DEFER_FLAG, @@ -19,8 +25,6 @@ DBT_SELECT_FLAG, DBT_UNIQUE_ID_SEPARATOR, ) -from sqlbuild.integrations.dbt.main.cli._build_command_argv import build_dbt_command_argv -from sqlbuild.integrations.dbt.main.cli._parse_ls_json_lines import parse_dbt_ls_json_lines from sqlbuild.integrations.dbt.main.runtime._execute_json_event_stream import ( execute_dbt_json_event_stream, ) diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py index 8dc4badeff..8cce2cdee5 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py @@ -14,6 +14,8 @@ build_dbt_combined_graph, discover_project_inputs, load_dbt_manifest_index, + resolve_dbt_executable, + route_dbt_interop_args, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.types import CompiledProjectView @@ -25,8 +27,6 @@ from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError from sqlbuild.integrations.dbt.main.cli._parse_execution_args import parse_dbt_execution_args -from sqlbuild.integrations.dbt.main.cli._resolve_executable import resolve_dbt_executable -from sqlbuild.integrations.dbt.main.cli._route_interop_args import route_dbt_interop_args from sqlbuild.integrations.dbt.main.config._resolve_plan_options import resolve_dbt_plan_options from sqlbuild.integrations.dbt.main.config._resolve_vars_mapping import resolve_dbt_vars_mapping from sqlbuild.integrations.dbt.main.manifest._resolve_manifest_path import ( diff --git a/src/sqlbuild/integrations/dbt/_helpers/planning/runtime.py b/src/sqlbuild/integrations/dbt/_helpers/planning/runtime.py index 05c78cdb96..18d3aeab87 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/planning/runtime.py +++ b/src/sqlbuild/integrations/dbt/_helpers/planning/runtime.py @@ -8,12 +8,12 @@ import yaml +from sqlbuild._native import resolve_dbt_config from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter from sqlbuild.adapter.discovery.main.builtin_adapter_classes import builtin_adapter_classes from sqlbuild.adapter.discovery.main.project_adapters import discover_project_adapters from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt._helpers.config.core import resolve_dbt_config from sqlbuild.integrations.dbt.constants import DBT_DEFER_FLAG from sqlbuild.integrations.dbt.exceptions import DbtInteropConfigError from sqlbuild.integrations.dbt.models import DbtCliConfigOverrides, DbtCliOptions, ResolvedDbtConfig diff --git a/src/sqlbuild/integrations/dbt/classes/dbt_runner.py b/src/sqlbuild/integrations/dbt/classes/dbt_runner.py index 93aa85d294..eff420c59b 100644 --- a/src/sqlbuild/integrations/dbt/classes/dbt_runner.py +++ b/src/sqlbuild/integrations/dbt/classes/dbt_runner.py @@ -7,13 +7,15 @@ from pathlib import Path from typing import cast +from sqlbuild._native import ( + build_dbt_compile_argv, + build_dbt_debug_argv, + build_dbt_deps_argv, + build_dbt_ls_argv, + parse_dbt_ls_json_lines, + resolve_dbt_executable, +) from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError -from sqlbuild.integrations.dbt.main.cli._build_compile_argv import build_dbt_compile_argv -from sqlbuild.integrations.dbt.main.cli._build_debug_argv import build_dbt_debug_argv -from sqlbuild.integrations.dbt.main.cli._build_deps_argv import build_dbt_deps_argv -from sqlbuild.integrations.dbt.main.cli._build_ls_argv import build_dbt_ls_argv -from sqlbuild.integrations.dbt.main.cli._parse_ls_json_lines import parse_dbt_ls_json_lines -from sqlbuild.integrations.dbt.main.cli._resolve_executable import resolve_dbt_executable from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtCommandResult, DbtLsResult from sqlbuild.integrations.dbt.types import DbtInvoker from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/integrations/dbt/constants.py b/src/sqlbuild/integrations/dbt/constants.py index c7f0a71beb..71ad0da810 100644 --- a/src/sqlbuild/integrations/dbt/constants.py +++ b/src/sqlbuild/integrations/dbt/constants.py @@ -14,9 +14,6 @@ DBT_MANIFEST_CONFIG_KEY: str = "config" DBT_MANIFEST_MATERIALIZED_KEY: str = "materialized" -DBT_EXECUTABLE_ENV_VAR: str = "DBT_EXECUTABLE" -DEFAULT_DBT_EXECUTABLE: str = "dbt" - DBT_DEFER_FLAG: str = "--defer" DBT_EXECUTION_FAIL_STATUSES: frozenset[str] = frozenset({"error", "fail", "failed"}) diff --git a/src/sqlbuild/integrations/dbt/main/cli/_build_command_argv.py b/src/sqlbuild/integrations/dbt/main/cli/_build_command_argv.py deleted file mode 100644 index 4e238a1bb5..0000000000 --- a/src/sqlbuild/integrations/dbt/main/cli/_build_command_argv.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Build dbt command arguments.""" - -from collections.abc import Sequence - -from sqlbuild.integrations.dbt._helpers.cli.runner import build_dbt_command_argv as _build -from sqlbuild.integrations.dbt.models import DbtCliOptions - - -def build_dbt_command_argv( - *, - dbt_executable: str, - command: str, - options: DbtCliOptions, - args: Sequence[str] = (), -) -> tuple[str, ...]: - """Build argv for an executable dbt command.""" - - return _build(dbt_executable=dbt_executable, command=command, options=options, args=args) diff --git a/src/sqlbuild/integrations/dbt/main/cli/_build_compile_argv.py b/src/sqlbuild/integrations/dbt/main/cli/_build_compile_argv.py deleted file mode 100644 index 7f6ea549fa..0000000000 --- a/src/sqlbuild/integrations/dbt/main/cli/_build_compile_argv.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Build dbt compile arguments.""" - -from sqlbuild.integrations.dbt._helpers.cli.runner import build_dbt_compile_argv as _build -from sqlbuild.integrations.dbt.models import DbtCliOptions - - -def build_dbt_compile_argv( - *, dbt_executable: str, options: DbtCliOptions, full_refresh: bool = False -) -> tuple[str, ...]: - """Build argv for dbt compile.""" - - return _build(dbt_executable=dbt_executable, options=options, full_refresh=full_refresh) diff --git a/src/sqlbuild/integrations/dbt/main/cli/_build_debug_argv.py b/src/sqlbuild/integrations/dbt/main/cli/_build_debug_argv.py deleted file mode 100644 index d9812a041f..0000000000 --- a/src/sqlbuild/integrations/dbt/main/cli/_build_debug_argv.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Build dbt debug arguments.""" - -from collections.abc import Sequence - -from sqlbuild.integrations.dbt._helpers.cli.runner import build_dbt_debug_argv as _build -from sqlbuild.integrations.dbt.models import DbtCliOptions - - -def build_dbt_debug_argv( - *, dbt_executable: str, options: DbtCliOptions, args: Sequence[str] = () -) -> tuple[str, ...]: - """Build argv for dbt debug.""" - - return _build(dbt_executable=dbt_executable, options=options, args=args) diff --git a/src/sqlbuild/integrations/dbt/main/cli/_build_deps_argv.py b/src/sqlbuild/integrations/dbt/main/cli/_build_deps_argv.py deleted file mode 100644 index 0e5b533d91..0000000000 --- a/src/sqlbuild/integrations/dbt/main/cli/_build_deps_argv.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Build dbt deps arguments.""" - -from sqlbuild.integrations.dbt._helpers.cli.runner import build_dbt_deps_argv as _build -from sqlbuild.integrations.dbt.models import DbtCliOptions - - -def build_dbt_deps_argv(*, dbt_executable: str, options: DbtCliOptions) -> tuple[str, ...]: - """Build argv for dbt deps.""" - - return _build(dbt_executable=dbt_executable, options=options) diff --git a/src/sqlbuild/integrations/dbt/main/cli/_build_ls_argv.py b/src/sqlbuild/integrations/dbt/main/cli/_build_ls_argv.py deleted file mode 100644 index 701a569dc3..0000000000 --- a/src/sqlbuild/integrations/dbt/main/cli/_build_ls_argv.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Build dbt ls arguments.""" - -from collections.abc import Sequence - -from sqlbuild.integrations.dbt._helpers.cli.runner import build_dbt_ls_argv as _build -from sqlbuild.integrations.dbt.models import DbtCliOptions - - -def build_dbt_ls_argv( - *, - dbt_executable: str, - options: DbtCliOptions, - select: Sequence[str] = (), - exclude: Sequence[str] = (), - resource_types: Sequence[str] = (), -) -> tuple[str, ...]: - """Build argv for dbt ls with JSON output.""" - - return _build( - dbt_executable=dbt_executable, - options=options, - select=select, - exclude=exclude, - resource_types=resource_types, - ) diff --git a/src/sqlbuild/integrations/dbt/main/cli/_parse_ls_json_lines.py b/src/sqlbuild/integrations/dbt/main/cli/_parse_ls_json_lines.py deleted file mode 100644 index 967db0c7b2..0000000000 --- a/src/sqlbuild/integrations/dbt/main/cli/_parse_ls_json_lines.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Parse dbt ls output.""" - -from sqlbuild.integrations.dbt._helpers.cli.runner import parse_dbt_ls_json_lines as _parse -from sqlbuild.integrations.dbt.models import DbtLsNode - - -def parse_dbt_ls_json_lines(*, stdout: str) -> tuple[DbtLsNode, ...]: - """Parse dbt ls JSON-lines output.""" - - return _parse(stdout=stdout) diff --git a/src/sqlbuild/integrations/dbt/main/cli/_resolve_executable.py b/src/sqlbuild/integrations/dbt/main/cli/_resolve_executable.py deleted file mode 100644 index 40929bb5cf..0000000000 --- a/src/sqlbuild/integrations/dbt/main/cli/_resolve_executable.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Resolve the dbt executable.""" - -from sqlbuild.integrations.dbt._helpers.cli.runner import resolve_dbt_executable as _resolve - - -def resolve_dbt_executable() -> str: - """Return the configured dbt executable.""" - - return _resolve() diff --git a/src/sqlbuild/integrations/dbt/main/cli/_route_interop_args.py b/src/sqlbuild/integrations/dbt/main/cli/_route_interop_args.py deleted file mode 100644 index aef9b6ca1e..0000000000 --- a/src/sqlbuild/integrations/dbt/main/cli/_route_interop_args.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Route dbt interop arguments.""" - -from sqlbuild.integrations.dbt._helpers.cli.args import route_dbt_interop_args as _route -from sqlbuild.integrations.dbt.models import DbtInteropParsedArgs, DbtInteropRoutedArgs -from sqlbuild.integrations.dbt.types import DbtInteropCommand - - -def route_dbt_interop_args( - *, command: DbtInteropCommand | str, parsed: DbtInteropParsedArgs -) -> DbtInteropRoutedArgs: - """Route parsed arguments to dbt and SQLBuild.""" - - return _route(command=command, parsed=parsed) diff --git a/src/sqlbuild/integrations/dbt/main/config/_resolve_config.py b/src/sqlbuild/integrations/dbt/main/config/_resolve_config.py deleted file mode 100644 index 115a890dd5..0000000000 --- a/src/sqlbuild/integrations/dbt/main/config/_resolve_config.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Resolve dbt configuration.""" - -from pathlib import Path - -from sqlbuild.integrations.dbt._helpers.config.core import resolve_dbt_config as _resolve -from sqlbuild.integrations.dbt.models import DbtCliConfigOverrides, ResolvedDbtConfig -from sqlbuild.spec.contracts.models import DbtConfig, LocalDbtConfig - - -def resolve_dbt_config( - *, - project_root: Path, - config: DbtConfig, - overrides: DbtCliConfigOverrides, - require_project_dir: bool, - local_config: LocalDbtConfig | None = None, -) -> ResolvedDbtConfig: - """Resolve dbt config from overrides and project configuration.""" - - return _resolve( - project_root=project_root, - config=config, - overrides=overrides, - require_project_dir=require_project_dir, - local_config=local_config, - ) diff --git a/src/sqlbuild/integrations/dbt/main/pipeline/debug.py b/src/sqlbuild/integrations/dbt/main/pipeline/debug.py index 8b49a3203f..6cb34b3b6b 100644 --- a/src/sqlbuild/integrations/dbt/main/pipeline/debug.py +++ b/src/sqlbuild/integrations/dbt/main/pipeline/debug.py @@ -5,13 +5,12 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import discover_project_inputs, resolve_dbt_config from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.main.config._parse_config_overrides import ( parse_dbt_config_overrides, ) -from sqlbuild.integrations.dbt.main.config._resolve_config import resolve_dbt_config from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtCommandResult, ResolvedDbtConfig diff --git a/src/sqlbuild/integrations/dbt/models.py b/src/sqlbuild/integrations/dbt/models.py index 4418aa8aa2..6635c9274a 100644 --- a/src/sqlbuild/integrations/dbt/models.py +++ b/src/sqlbuild/integrations/dbt/models.py @@ -271,7 +271,6 @@ class DbtLsNode: name: str | None = None fqn: tuple[str, ...] = field(default_factory=tuple) original_file_path: str | None = None - payload: dict[str, object] = field(default_factory=dict) @property def selector_term(self) -> str: diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index 7d94e0304e..b5c3000671 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -13,6 +13,7 @@ DbtEventStreamUpdate, assemble_project, build_dbt_combined_graph, + build_dbt_ls_argv, build_dbt_manifest_index, extract_sql_references, transitive_closure, @@ -37,7 +38,6 @@ PlanReason, ) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.integrations.dbt._helpers.cli.runner import build_dbt_ls_argv from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.models import ( DbtCliConfigOverrides, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_args.py b/tests/unit/src/sqlbuild/integrations/dbt/test_args.py index 71ca92ed72..ed503a1a62 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_args.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_args.py @@ -2,8 +2,8 @@ import pytest +from sqlbuild._native import route_dbt_interop_args from sqlbuild.integrations.dbt._helpers.cli.arg_parser import parse_dbt_execution_args -from sqlbuild.integrations.dbt._helpers.cli.args import route_dbt_interop_args from sqlbuild.integrations.dbt.exceptions import DbtInteropArgumentError from sqlbuild.integrations.dbt.models import DbtInteropParsedArgs, DbtInteropRoutedArgs from sqlbuild.integrations.dbt.types import DbtInteropCommand diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_config.py b/tests/unit/src/sqlbuild/integrations/dbt/test_config.py index 2a3d4432bd..39f04923f5 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_config.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_config.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild.integrations.dbt._helpers.config.core import resolve_dbt_config +from sqlbuild._native import resolve_dbt_config from sqlbuild.integrations.dbt._helpers.planning.runtime import resolve_dbt_vars from sqlbuild.integrations.dbt.exceptions import DbtInteropConfigError from sqlbuild.integrations.dbt.models import ResolvedDbtConfig diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py b/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py index 7af5b5b789..690374359e 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py @@ -4,9 +4,13 @@ import pytest -from sqlbuild._native import DbtCombinedGraph, build_dbt_combined_graph, build_dbt_manifest_index +from sqlbuild._native import ( + DbtCombinedGraph, + build_dbt_combined_graph, + build_dbt_ls_argv, + build_dbt_manifest_index, +) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.integrations.dbt._helpers.cli.runner import build_dbt_ls_argv from sqlbuild.integrations.dbt._helpers.planning.orchestration import plan_dbt_interop_command from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_runner.py b/tests/unit/src/sqlbuild/integrations/dbt/test_runner.py index 837b3a1ca9..caae1125c2 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_runner.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_runner.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.integrations.dbt._helpers.cli.runner import ( +from sqlbuild._native import ( build_dbt_compile_argv, build_dbt_debug_argv, build_dbt_deps_argv, From fc5db683a9b55e8c7b1519e4c246bc71486f4a66 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:25:31 +0100 Subject: [PATCH 22/24] refactor: plan sqb dbt commands natively --- .../dbt_plan/main/dbt_anchor_term_check.rs | 17 ++ .../src/dbt_plan/main/dbt_failure_detail.rs | 11 + .../src/dbt_plan/main/dbt_plan_argvs.rs | 48 ++++ .../main/dbt_required_selector_terms.rs | 21 ++ crates/sqlbuild-core/src/dbt_plan/main/mod.rs | 4 + crates/sqlbuild-core/src/dbt_plan/mod.rs | 7 + crates/sqlbuild-core/src/dbt_plan/models.rs | 32 +++ crates/sqlbuild-core/src/dbt_plan/tests.rs | 6 + .../src/dbt_plan/tests/helpers.rs | 13 + .../src/dbt_plan/tests/test_dbt_plan.rs | 148 +++++++++++ .../src/dbt_plan/tests/test_types.rs | 11 + crates/sqlbuild-core/src/lib.rs | 1 + .../src/bindings/_helpers/dbt_interop/cli.rs | 4 +- .../bindings/_helpers/dbt_interop/graph.rs | 30 ++- .../src/bindings/_helpers/dbt_interop/mod.rs | 1 + .../_helpers/dbt_interop/orchestration.rs | 236 ++++++++++++++++++ .../src/bindings/_helpers/dbt_interop/plan.rs | 2 +- .../_helpers/dbt_interop/registration.rs | 3 +- src/sqlbuild/_native.pyi | 12 + .../dbt/_helpers/pipeline/interop_prologue.py | 2 +- .../dbt/_helpers/planning/orchestration.py | 227 ----------------- src/sqlbuild/integrations/dbt/constants.py | 1 - .../dbt/main/planning/__init__.py | 1 - .../main/planning/_plan_interop_command.py | 38 --- .../dbt/test_plan_orchestration.py | 2 +- 25 files changed, 593 insertions(+), 285 deletions(-) create mode 100644 crates/sqlbuild-core/src/dbt_plan/main/dbt_anchor_term_check.rs create mode 100644 crates/sqlbuild-core/src/dbt_plan/main/dbt_failure_detail.rs create mode 100644 crates/sqlbuild-core/src/dbt_plan/main/dbt_plan_argvs.rs create mode 100644 crates/sqlbuild-core/src/dbt_plan/main/dbt_required_selector_terms.rs create mode 100644 crates/sqlbuild-core/src/dbt_plan/main/mod.rs create mode 100644 crates/sqlbuild-core/src/dbt_plan/mod.rs create mode 100644 crates/sqlbuild-core/src/dbt_plan/models.rs create mode 100644 crates/sqlbuild-core/src/dbt_plan/tests.rs create mode 100644 crates/sqlbuild-core/src/dbt_plan/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/dbt_plan/tests/test_dbt_plan.rs create mode 100644 crates/sqlbuild-core/src/dbt_plan/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs delete mode 100644 src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py delete mode 100644 src/sqlbuild/integrations/dbt/main/planning/__init__.py delete mode 100644 src/sqlbuild/integrations/dbt/main/planning/_plan_interop_command.py diff --git a/crates/sqlbuild-core/src/dbt_plan/main/dbt_anchor_term_check.rs b/crates/sqlbuild-core/src/dbt_plan/main/dbt_anchor_term_check.rs new file mode 100644 index 0000000000..33614eb200 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_plan/main/dbt_anchor_term_check.rs @@ -0,0 +1,17 @@ +//! Whether a selector term anchors SQLBuild work on dbt nodes. + +use crate::dbt_graph::constants::PATH_SELECTOR_SEPARATOR; +use crate::dbt_graph::models::DbtSelectorExpansion; +use crate::dbt_graph::types::DbtSelectorResolver; + +/// A trailing-plus term that is neither a path selector nor a SQLBuild model selector. +pub fn dbt_anchor_term_check( + resolver: &impl DbtSelectorResolver, + term: &str, +) -> Result { + let parsed: DbtSelectorExpansion = resolver.split(term)?; + if !parsed.downstream || parsed.core.contains(PATH_SELECTOR_SEPARATOR) { + return Ok(false); + } + Ok(resolver.model_names(&parsed.core)?.0.is_empty()) +} diff --git a/crates/sqlbuild-core/src/dbt_plan/main/dbt_failure_detail.rs b/crates/sqlbuild-core/src/dbt_plan/main/dbt_failure_detail.rs new file mode 100644 index 0000000000..e6c89220bf --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_plan/main/dbt_failure_detail.rs @@ -0,0 +1,11 @@ +//! The detail shown for a failed dbt command. + +use crate::text::main::python_strip::python_strip; + +/// Stripped stderr, or stdout when stderr is empty; `None` when nothing remains. +#[must_use] +pub fn dbt_failure_detail(stderr: &str, stdout: &str) -> Option { + let output: &str = if stderr.is_empty() { stdout } else { stderr }; + let detail: &str = python_strip(output); + (!detail.is_empty()).then(|| detail.to_owned()) +} diff --git a/crates/sqlbuild-core/src/dbt_plan/main/dbt_plan_argvs.rs b/crates/sqlbuild-core/src/dbt_plan/main/dbt_plan_argvs.rs new file mode 100644 index 0000000000..79d45daab1 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_plan/main/dbt_plan_argvs.rs @@ -0,0 +1,48 @@ +//! The plan's dbt and SQLBuild command lines. + +use crate::dbt_cli::main::build_dbt_argv::build_dbt_argv; +use crate::dbt_cli::models::{DbtArgvCommand, DbtInteropCommandKind}; +use crate::dbt_plan::models::{DbtPlanArgvRequest, DbtPlanArgvs}; + +/// `dbt ls` for plans, the command itself otherwise; SQLBuild runs models with `build` minus tests. +#[must_use] +pub fn dbt_plan_argvs(request: &DbtPlanArgvRequest<'_>) -> DbtPlanArgvs { + let dbt_command: &str = match request.command { + DbtInteropCommandKind::Plan => "ls", + command => command.name(), + }; + let mut dbt_command_argv: Vec = + vec![request.dbt_executable.to_owned(), dbt_command.to_owned()]; + dbt_command_argv.extend(request.dbt_command_args.iter().cloned()); + let mut sqlbuild_command_argvs: Vec> = Vec::new(); + if !request.selected_model_names.is_empty() { + let sqlbuild_command: &[&str] = match request.command { + DbtInteropCommandKind::Run => &["build", "--no-tests", "--no-audits"], + command => &[command.name()], + }; + let mut argv: Vec = vec![request.sqlbuild_executable.to_owned()]; + argv.extend(sqlbuild_command.iter().map(|part| (*part).to_owned())); + argv.push("--select".to_owned()); + argv.extend(request.selected_model_names.iter().cloned()); + argv.extend(request.sqlbuild_command_args.iter().cloned()); + sqlbuild_command_argvs.push(argv); + } + let mut supplemental_dbt_command_argvs: Vec> = Vec::new(); + if !request.required_selector_terms.is_empty() { + let mut args: Vec = vec!["--select".to_owned()]; + args.extend(request.required_selector_terms.iter().cloned()); + supplemental_dbt_command_argvs.push(build_dbt_argv( + request.dbt_executable, + request.options, + DbtArgvCommand::Other { + command: dbt_command.to_owned(), + args, + }, + )); + } + DbtPlanArgvs { + dbt_command_argv, + sqlbuild_command_argvs, + supplemental_dbt_command_argvs, + } +} diff --git a/crates/sqlbuild-core/src/dbt_plan/main/dbt_required_selector_terms.rs b/crates/sqlbuild-core/src/dbt_plan/main/dbt_required_selector_terms.rs new file mode 100644 index 0000000000..e56378e3e4 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_plan/main/dbt_required_selector_terms.rs @@ -0,0 +1,21 @@ +//! dbt selectors for the dbt models selected SQLBuild models need. + +use std::collections::{BTreeSet, HashSet}; + +use crate::dbt_graph::main::dbt_fqn_selector_term::dbt_fqn_selector_term; +use crate::dbt_plan::models::DbtReferencedModel; + +/// Sorted, distinct `+` terms for referenced dbt models that are required. +#[must_use] +pub fn dbt_required_selector_terms( + referenced: &[DbtReferencedModel], + required_unique_ids: &[String], +) -> Vec { + let required: HashSet<&str> = required_unique_ids.iter().map(String::as_str).collect(); + let terms: BTreeSet = referenced + .iter() + .filter(|model| required.contains(model.unique_id.as_str())) + .map(|model| format!("+{}", dbt_fqn_selector_term(&model.fqn, &model.name))) + .collect(); + terms.into_iter().collect() +} diff --git a/crates/sqlbuild-core/src/dbt_plan/main/mod.rs b/crates/sqlbuild-core/src/dbt_plan/main/mod.rs new file mode 100644 index 0000000000..9104e5cb9e --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_plan/main/mod.rs @@ -0,0 +1,4 @@ +pub mod dbt_anchor_term_check; +pub mod dbt_failure_detail; +pub mod dbt_plan_argvs; +pub mod dbt_required_selector_terms; diff --git a/crates/sqlbuild-core/src/dbt_plan/mod.rs b/crates/sqlbuild-core/src/dbt_plan/mod.rs new file mode 100644 index 0000000000..7452d2ad1b --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_plan/mod.rs @@ -0,0 +1,7 @@ +//! `sqb dbt` plan and execution decisions: command lines, required dbt selectors and anchors. + +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/dbt_plan/models.rs b/crates/sqlbuild-core/src/dbt_plan/models.rs new file mode 100644 index 0000000000..74d6a6c9d0 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_plan/models.rs @@ -0,0 +1,32 @@ +//! Plan command-line inputs and outputs. + +use crate::dbt_cli::models::{DbtCliOptionValues, DbtInteropCommandKind}; + +/// What the plan's dbt and SQLBuild command lines are built from. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DbtPlanArgvRequest<'a> { + pub command: DbtInteropCommandKind, + pub dbt_executable: &'a str, + pub sqlbuild_executable: &'a str, + pub options: &'a DbtCliOptionValues, + pub dbt_command_args: &'a [String], + pub sqlbuild_command_args: &'a [String], + pub selected_model_names: &'a [String], + pub required_selector_terms: &'a [String], +} + +/// The primary dbt command, the SQLBuild commands and the supplemental dbt commands. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtPlanArgvs { + pub dbt_command_argv: Vec, + pub sqlbuild_command_argvs: Vec>, + pub supplemental_dbt_command_argvs: Vec>, +} + +/// One dbt model a selected SQLBuild model references. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DbtReferencedModel { + pub unique_id: String, + pub fqn: Vec, + pub name: String, +} diff --git a/crates/sqlbuild-core/src/dbt_plan/tests.rs b/crates/sqlbuild-core/src/dbt_plan/tests.rs new file mode 100644 index 0000000000..c333bb6e0a --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_plan/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_dbt_plan.rs"] +mod dbt_plan; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/dbt_plan/tests/helpers.rs b/crates/sqlbuild-core/src/dbt_plan/tests/helpers.rs new file mode 100644 index 0000000000..2945e2404d --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_plan/tests/helpers.rs @@ -0,0 +1,13 @@ +use crate::dbt_plan::models::DbtPlanArgvs; + +pub(super) fn texts(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// The primary, SQLBuild and supplemental command lines in order. +pub(super) fn flattened(argvs: DbtPlanArgvs) -> Vec> { + let mut all: Vec> = vec![argvs.dbt_command_argv]; + all.extend(argvs.sqlbuild_command_argvs); + all.extend(argvs.supplemental_dbt_command_argvs); + all +} diff --git a/crates/sqlbuild-core/src/dbt_plan/tests/test_dbt_plan.rs b/crates/sqlbuild-core/src/dbt_plan/tests/test_dbt_plan.rs new file mode 100644 index 0000000000..0f336a250d --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_plan/tests/test_dbt_plan.rs @@ -0,0 +1,148 @@ +use crate::dbt_cli::models::{DbtCliOptionValues, DbtInteropCommandKind}; +use crate::dbt_plan::main::dbt_failure_detail::dbt_failure_detail; +use crate::dbt_plan::main::dbt_plan_argvs::dbt_plan_argvs; +use crate::dbt_plan::main::dbt_required_selector_terms::dbt_required_selector_terms; +use crate::dbt_plan::models::{DbtPlanArgvRequest, DbtReferencedModel}; +use crate::dbt_plan::tests::helpers::{flattened, texts}; +use crate::dbt_plan::tests::test_types::{ArgvsTestCase, TextsTestCase}; + +#[test] +fn given_plan_inputs_when_building_argvs_then_maps_commands() { + let options: DbtCliOptionValues = DbtCliOptionValues { + target: Some("prod".to_owned()), + defer: true, + ..DbtCliOptionValues::default() + }; + let (dbt_args, sqlbuild_args, models, terms) = ( + texts(&["--select", "a+"]), + texts(&["--vars", "{}"]), + texts(&["fact", "mart"]), + texts(&["+fqn:a.stg"]), + ); + let request = |command: DbtInteropCommandKind, models: &'static [String]| DbtPlanArgvRequest { + command, + dbt_executable: "dbt", + sqlbuild_executable: "sqb", + options: &options, + dbt_command_args: &dbt_args, + sqlbuild_command_args: &sqlbuild_args, + selected_model_names: models, + required_selector_terms: &terms, + }; + let test_cases = [ + ArgvsTestCase { + description: "plan lists with dbt ls and plans SQLBuild", + argvs: flattened(dbt_plan_argvs(&DbtPlanArgvRequest { + selected_model_names: &models, + ..request(DbtInteropCommandKind::Plan, &[]) + })), + expected_argvs: vec![ + vec!["dbt", "ls", "--select", "a+"], + vec!["sqb", "plan", "--select", "fact", "mart", "--vars", "{}"], + vec![ + "dbt", + "ls", + "--target", + "prod", + "--defer", + "--select", + "+fqn:a.stg", + ], + ], + }, + ArgvsTestCase { + description: "run builds SQLBuild models without tests", + argvs: flattened(dbt_plan_argvs(&DbtPlanArgvRequest { + selected_model_names: &models, + required_selector_terms: &[], + ..request(DbtInteropCommandKind::Run, &[]) + })), + expected_argvs: vec![ + vec!["dbt", "run", "--select", "a+"], + vec![ + "sqb", + "build", + "--no-tests", + "--no-audits", + "--select", + "fact", + "mart", + "--vars", + "{}", + ], + ], + }, + ArgvsTestCase { + description: "no selected models means no SQLBuild command", + argvs: flattened(dbt_plan_argvs(&request(DbtInteropCommandKind::Build, &[]))), + expected_argvs: vec![ + vec!["dbt", "build", "--select", "a+"], + vec![ + "dbt", + "build", + "--target", + "prod", + "--defer", + "--select", + "+fqn:a.stg", + ], + ], + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_case.argvs, test_case.expected_argvs, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_dbt_outputs_when_summarising_then_returns_terms_and_details() { + let referenced: Vec = vec![ + DbtReferencedModel { + unique_id: "model.a.stg".to_owned(), + fqn: texts(&["a", "stg"]), + name: "stg".to_owned(), + }, + DbtReferencedModel { + unique_id: "model.a.int".to_owned(), + fqn: Vec::new(), + name: "int".to_owned(), + }, + DbtReferencedModel { + unique_id: "model.a.stg".to_owned(), + fqn: texts(&["a", "stg"]), + name: "stg".to_owned(), + }, + ]; + let test_cases = [ + TextsTestCase { + description: "required terms are distinct and sorted", + texts: dbt_required_selector_terms( + &referenced, + &texts(&["model.a.stg", "model.a.int"]), + ) + .into_iter() + .map(Some) + .collect(), + expected_texts: vec![Some("+fqn:a.stg"), Some("+int")], + }, + TextsTestCase { + description: "failure detail prefers stderr and drops blanks", + texts: vec![ + dbt_failure_detail(" boom \n", "out"), + dbt_failure_detail("", " out "), + dbt_failure_detail("", " \n"), + ], + expected_texts: vec![Some("boom"), Some("out"), None], + }, + ]; + + for test_case in test_cases { + let texts: Vec> = test_case.texts.iter().map(Option::as_deref).collect(); + assert_eq!(texts, test_case.expected_texts, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-core/src/dbt_plan/tests/test_types.rs b/crates/sqlbuild-core/src/dbt_plan/tests/test_types.rs new file mode 100644 index 0000000000..dd5580f9e7 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_plan/tests/test_types.rs @@ -0,0 +1,11 @@ +pub(super) struct ArgvsTestCase { + pub(super) description: &'static str, + pub(super) argvs: Vec>, + pub(super) expected_argvs: Vec>, +} + +pub(super) struct TextsTestCase { + pub(super) description: &'static str, + pub(super) texts: Vec>, + pub(super) expected_texts: Vec>, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 277f12461b..e143b3605c 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -15,6 +15,7 @@ pub mod dbt_cli; pub mod dbt_command; pub mod dbt_events; pub mod dbt_graph; +pub mod dbt_plan; pub mod debug_report; pub mod format_command; pub mod freshness_text; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs index 65c4d9a2f5..534a637084 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs @@ -52,7 +52,7 @@ fn texts(values: &Bound<'_, PyAny>) -> PyResult> { values.try_iter()?.map(|value| value?.extract()).collect() } -fn option_values(options: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn option_values(options: &Bound<'_, PyAny>) -> PyResult { Ok(DbtCliOptionValues { project_dir: attribute_text(options, "project_dir")?, profiles_dir: attribute_text(options, "profiles_dir")?, @@ -236,7 +236,7 @@ fn parse_dbt_ls_json_lines<'py>(py: Python<'py>, stdout: &str) -> PyResult, class_name: &str, error: DbtCliError) -> PyErr { +pub(crate) fn raised(py: Python<'_>, class_name: &str, error: DbtCliError) -> PyErr { let created = || -> PyResult { let keywords: Bound<'_, PyDict> = PyDict::new(py); if let Some(code) = error.code { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs index 7f332e870a..681478de22 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs @@ -91,7 +91,7 @@ impl DbtCombinedGraph { } } -fn texts(values: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn texts(values: &Bound<'_, PyAny>) -> PyResult> { values.try_iter()?.map(|value| value?.extract()).collect() } @@ -173,12 +173,26 @@ fn build_dbt_combined_graph<'py>( }) } -struct PlannerResolver<'py> { +pub(crate) struct PlannerResolver<'py> { project: Bound<'py, PyAny>, split: Bound<'py, PyAny>, resolve: Bound<'py, PyAny>, } +/// The planner's selector parsing and SQLBuild model selector resolution over `project`. +pub(crate) fn planner_resolver(project: Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = project.py(); + Ok(PlannerResolver { + split: imported(py, SELECTOR_EXPANSION_MODULE, "split_selector_expansion")?, + resolve: imported( + py, + MODEL_SELECTORS_MODULE, + "resolve_sqlbuild_model_selector_names", + )?, + project, + }) +} + impl DbtSelectorResolver for PlannerResolver<'_> { type Error = PyErr; @@ -283,7 +297,7 @@ fn selection_result<'py>(py: Python<'py>, selection: DbtSelection) -> PyResult( +pub(crate) fn resolve_dbt_interop_sqlbuild_selection<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -311,15 +325,7 @@ fn resolve_dbt_interop_sqlbuild_selection<'py>( manifest_models: manifest_models.as_deref(), }; let graph: Bound<'py, DbtCombinedGraph> = arguments.value("graph")?.cast_into()?; - let resolver: PlannerResolver<'py> = PlannerResolver { - project: arguments.value("project")?, - split: imported(py, SELECTOR_EXPANSION_MODULE, "split_selector_expansion")?, - resolve: imported( - py, - MODEL_SELECTORS_MODULE, - "resolve_sqlbuild_model_selector_names", - )?, - }; + let resolver: PlannerResolver<'py> = planner_resolver(arguments.value("project")?)?; let selection: DbtSelection = resolve_dbt_selection(&resolver, &graph.get().graph, &request) .map_err(|error| selection_error(py, error))?; selection_result(py, selection) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs index 4046369c14..06322ff9e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod cli; pub(crate) mod graph; pub(crate) mod manifest; +pub(crate) mod orchestration; pub(crate) mod plan; pub(crate) mod profile; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs new file mode 100644 index 0000000000..7ca16f2b71 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs @@ -0,0 +1,236 @@ +//! Plan one `sqb dbt` command from compiled inputs: dbt listings, anchors, selection and argvs. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::dbt_cli::errors::DbtCliError; +use sqlbuild_core::dbt_cli::models::{DbtCliOptionValues, DbtInteropCommandKind}; +use sqlbuild_core::dbt_plan::main::dbt_anchor_term_check::dbt_anchor_term_check; +use sqlbuild_core::dbt_plan::main::dbt_failure_detail::dbt_failure_detail; +use sqlbuild_core::dbt_plan::main::dbt_plan_argvs::dbt_plan_argvs; +use sqlbuild_core::dbt_plan::main::dbt_required_selector_terms::dbt_required_selector_terms; +use sqlbuild_core::dbt_plan::models::{DbtPlanArgvRequest, DbtPlanArgvs, DbtReferencedModel}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, +}; +use crate::bindings::_helpers::dbt_interop::cli::{option_values, raised}; +use crate::bindings::_helpers::dbt_interop::graph::{ + PlannerResolver, planner_resolver, resolve_dbt_interop_sqlbuild_selection, texts, +}; +use crate::bindings::_helpers::dbt_interop::manifest::resolve_sqlbuild_model_dbt_refs; +use crate::bindings::_helpers::dbt_interop::plan::build_dbt_interop_plan; + +const DBT_TYPES: &str = "sqlbuild.integrations.dbt.types"; +const PLAN_OPTIONS: (&[&str], &[&str]) = ( + &[ + "command", + "project", + "manifest", + "graph", + "dbt_runner", + "dbt_options", + "command_args", + ], + &[], +); + +/// `dbt_runner.ls(...)` result, raising when dbt ls failed. +fn listed<'py>( + (runner, dbt_options): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + select: Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = runner.py(); + let result: Bound<'py, PyAny> = call_with( + &runner.getattr("ls")?, + &[ + ("options", dbt_options.clone()), + ("select", select), + ("exclude", exclude.clone()), + ], + )?; + let command: Bound<'py, PyAny> = result.getattr("command")?; + if command.getattr("returncode")?.extract::()? == 0 { + return Ok(result); + } + let detail: Option = dbt_failure_detail( + &command.getattr("stderr")?.extract::()?, + &command.getattr("stdout")?.extract::()?, + ); + let help: Bound<'py, PyAny> = detail.map_or_else( + || py.None().into_bound(py), + |detail| PyString::new(py, &detail).into_any(), + ); + let error: Bound<'py, PyAny> = imported( + py, + "sqlbuild.integrations.dbt.exceptions", + "DbtInteropRuntimeError", + )? + .call(("dbt ls failed",), Some(&keywords(py, &[("help", help)])?))?; + Err(PyErr::from_value(error)) +} + +fn unique_ids<'py>(listing: &Bound<'py, PyAny>) -> PyResult> { + let mut ids: Vec> = Vec::new(); + for node in listing.getattr("nodes")?.try_iter()? { + ids.push(node?.getattr("unique_id")?); + } + PyTuple::new(listing.py(), ids) +} + +fn command_kind(command: &Bound<'_, PyAny>) -> PyResult { + let value: String = command.getattr("value")?.extract()?; + match value.as_str() { + "plan" => Ok(DbtInteropCommandKind::Plan), + "run" => Ok(DbtInteropCommandKind::Run), + "build" => Ok(DbtInteropCommandKind::Build), + "debug" => Ok(DbtInteropCommandKind::Debug), + _ => Err(raised( + command.py(), + "DbtInteropArgumentError", + DbtCliError { + message: format!("Unsupported sqb dbt command: {value}"), + ..DbtCliError::default() + }, + )), + } +} + +fn required_terms( + arguments: &KeywordArguments<'_>, + selection: &Bound<'_, PyAny>, +) -> PyResult> { + let required: Vec = texts(&selection.getattr("dbt_required_unique_ids")?)?; + if required.is_empty() { + return Ok(Vec::new()); + } + let py: Python<'_> = selection.py(); + let mut referenced: Vec = Vec::new(); + for pair in resolve_sqlbuild_model_dbt_refs( + py, + arguments.value("project")?, + arguments.value("manifest")?, + Some(selection.getattr("sqlbuild_model_names")?), + )? + .try_iter()? + { + let (_model, dbt_model): (Bound<'_, PyAny>, Bound<'_, PyAny>) = pair?.extract()?; + referenced.push(DbtReferencedModel { + unique_id: dbt_model.getattr("unique_id")?.extract()?, + fqn: texts(&dbt_model.getattr("fqn")?)?, + name: dbt_model.getattr("name")?.extract()?, + }); + } + Ok(dbt_required_selector_terms(&referenced, &required)) +} + +fn argv_tuples<'py>(py: Python<'py>, argvs: Vec>) -> PyResult> { + let tuples: Vec> = argvs + .into_iter() + .map(|argv| PyTuple::new(py, argv)) + .collect::>()?; + Ok(PyTuple::new(py, tuples)?.into_any()) +} + +/// Plan dbt and SQLBuild work from already-compiled interop inputs. +#[pyfunction] +#[pyo3(signature = (**options))] +fn plan_dbt_interop_command<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "plan_dbt_interop_command", options, PLAN_OPTIONS)?; + let command: Bound<'py, PyAny> = + imported(py, DBT_TYPES, "DbtInteropCommand")?.call1((arguments.value("command")?,))?; + let command_args: Bound<'py, PyAny> = arguments.value("command_args")?; + let (select, exclude) = ( + command_args.getattr("select")?, + command_args.getattr("exclude")?, + ); + let (runner, dbt_options) = ( + arguments.value("dbt_runner")?, + arguments.value("dbt_options")?, + ); + let full_listing: Bound<'py, PyAny> = + listed((&runner, &dbt_options), select.clone(), &exclude)?; + let anchors: Bound<'py, PyDict> = PyDict::new(py); + let resolver: PlannerResolver<'py> = planner_resolver(arguments.value("project")?)?; + for term in texts(&select)? { + if !dbt_anchor_term_check(&resolver, &term)? { + continue; + } + let only_term: Bound<'py, PyAny> = PyTuple::new(py, [&term])?.into_any(); + let listing: Bound<'py, PyAny> = listed((&runner, &dbt_options), only_term, &exclude)?; + anchors.set_item(&term, unique_ids(&listing)?)?; + } + let selection: Bound<'py, PyAny> = resolve_dbt_interop_sqlbuild_selection( + py, + Some(keywords( + py, + &[ + ("project", arguments.value("project")?), + ("manifest", arguments.value("manifest")?), + ("graph", arguments.value("graph")?), + ("select", select.clone()), + ("exclude", exclude.clone()), + ("dbt_anchor_unique_ids_by_term", anchors.into_any()), + ], + )?), + )?; + let required_selector_terms: Vec = required_terms(&arguments, &selection)?; + let options: DbtCliOptionValues = option_values(&dbt_options)?; + let (dbt_executable, sqlbuild_executable): (String, String) = ( + command_args.getattr("dbt_executable")?.extract()?, + command_args.getattr("sqlbuild_executable")?.extract()?, + ); + let (dbt_command_args, sqlbuild_command_args, selected_model_names) = ( + texts(&command_args.getattr("dbt_command_args")?)?, + texts(&command_args.getattr("sqlbuild_command_args")?)?, + texts(&selection.getattr("sqlbuild_model_names")?)?, + ); + let argvs: DbtPlanArgvs = dbt_plan_argvs(&DbtPlanArgvRequest { + command: command_kind(&command)?, + dbt_executable: &dbt_executable, + sqlbuild_executable: &sqlbuild_executable, + options: &options, + dbt_command_args: &dbt_command_args, + sqlbuild_command_args: &sqlbuild_command_args, + selected_model_names: &selected_model_names, + required_selector_terms: &required_selector_terms, + }); + build_dbt_interop_plan( + py, + Some(keywords( + py, + &[ + ("command", command), + ( + "dbt_command_argv", + PyTuple::new(py, argvs.dbt_command_argv)?.into_any(), + ), + ("dbt_ls_nodes", full_listing.getattr("nodes")?), + ( + "sqlbuild_command_argvs", + argv_tuples(py, argvs.sqlbuild_command_argvs)?, + ), + ("selection", selection), + ( + "dbt_required_selector_terms", + PyTuple::new(py, required_selector_terms)?.into_any(), + ), + ( + "supplemental_dbt_command_argvs", + argv_tuples(py, argvs.supplemental_dbt_command_argvs)?, + ), + ], + )?), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(plan_dbt_interop_command, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs index 82c1cbc183..5d2fd1314f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs @@ -130,7 +130,7 @@ fn skip_reason<'py>(py: Python<'py>, has_work: bool, member: &str) -> PyResult( +pub(crate) fn build_dbt_interop_plan<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs index 439de2d4c3..0aed90bc7f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs @@ -2,12 +2,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::dbt_interop::{cli, graph, manifest, plan, profile}; +use crate::bindings::_helpers::dbt_interop::{cli, graph, manifest, orchestration, plan, profile}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { cli::register(module)?; graph::register(module)?; manifest::register(module)?; + orchestration::register(module)?; plan::register(module)?; profile::register(module) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 87942ca75b..a6a8e948f5 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -260,9 +260,11 @@ from sqlbuild.executor.python_nodes.models import ( ) from sqlbuild.executor.scenario.models import ScenarioRunResult, ScenarioSnapshotCaptureRunResult from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult +from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.models import ( DbtCliConfigOverrides, DbtCliOptions, + DbtInteropCommandArgs, DbtInteropParsedArgs, DbtInteropPlan, DbtInteropRoutedArgs, @@ -1490,6 +1492,16 @@ def resolve_dbt_interop_sqlbuild_selection( dbt_anchor_unique_ids_by_term: Mapping[str, Sequence[str]] | None = None, ) -> DbtInteropSelectionResult: ... def dbt_fqn_selector_term(*, fqn: Sequence[str], fallback: str) -> str: ... +def plan_dbt_interop_command( + *, + command: DbtInteropCommand | str, + project: CompiledProjectView, + manifest: DbtManifestIndex, + graph: DbtCombinedGraph, + dbt_runner: DbtRunner, + dbt_options: DbtCliOptions, + command_args: DbtInteropCommandArgs, +) -> DbtInteropPlan: ... def resolve_dbt_executable() -> str: ... def build_dbt_compile_argv( *, dbt_executable: str, options: DbtCliOptions, full_refresh: bool = False diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py index 8cce2cdee5..10ab3b2a7f 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py @@ -14,6 +14,7 @@ build_dbt_combined_graph, discover_project_inputs, load_dbt_manifest_index, + plan_dbt_interop_command, resolve_dbt_executable, route_dbt_interop_args, ) @@ -32,7 +33,6 @@ from sqlbuild.integrations.dbt.main.manifest._resolve_manifest_path import ( resolve_dbt_manifest_path, ) -from sqlbuild.integrations.dbt.main.planning._plan_interop_command import plan_dbt_interop_command from sqlbuild.integrations.dbt.main.runtime._report_progress import report_progress from sqlbuild.integrations.dbt.main.runtime._resolve_interop_adapter import ( resolve_dbt_interop_adapter, diff --git a/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py b/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py deleted file mode 100644 index 53356ac554..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py +++ /dev/null @@ -1,227 +0,0 @@ -"""Pure orchestration for dbt interop plan inputs.""" - -from __future__ import annotations - -from collections.abc import Sequence - -from sqlbuild._native import ( - DbtCombinedGraph, - build_dbt_interop_plan, - dbt_fqn_selector_term, - resolve_dbt_interop_sqlbuild_selection, - resolve_sqlbuild_model_dbt_refs, -) -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.main.selection.selector_expansion import split_selector_expansion -from sqlbuild.compiler.planner.main.selection.sqlbuild_model_selectors import ( - resolve_sqlbuild_model_selector_names, -) -from sqlbuild.compiler.planner.models import SelectorExpansion -from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner -from sqlbuild.integrations.dbt.constants import DBT_PATH_SELECTOR_SEPARATOR -from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError -from sqlbuild.integrations.dbt.models import ( - DbtCliOptions, - DbtInteropCommandArgs, - DbtInteropPlan, - DbtInteropSelectionResult, - DbtLsResult, - DbtManifestIndex, -) -from sqlbuild.integrations.dbt.types import DbtInteropCommand - - -def plan_dbt_interop_command( - *, - command: DbtInteropCommand | str, - project: CompiledProjectView, - manifest: DbtManifestIndex, - graph: DbtCombinedGraph, - dbt_runner: DbtRunner, - dbt_options: DbtCliOptions, - command_args: DbtInteropCommandArgs, -) -> DbtInteropPlan: - """Plan dbt and SQLBuild work from already-compiled interop inputs.""" - - select: Sequence[str] = command_args.select - exclude: Sequence[str] = command_args.exclude - dbt_command_args: Sequence[str] = command_args.dbt_command_args - sqlbuild_command_args: Sequence[str] = command_args.sqlbuild_command_args - dbt_executable: str = command_args.dbt_executable - sqlbuild_executable: str = command_args.sqlbuild_executable - normalized_command: DbtInteropCommand = DbtInteropCommand(command) - full_dbt_ls: DbtLsResult = dbt_runner.ls( - options=dbt_options, - select=select, - exclude=exclude, - ) - _raise_for_dbt_ls_failure(full_dbt_ls) - anchors_by_term: dict[str, tuple[str, ...]] = {} - term: str - for term in select: - if not _is_dbt_anchor_term(term=term, project=project): - continue - anchor_ls: DbtLsResult = dbt_runner.ls( - options=dbt_options, - select=(term,), - exclude=exclude, - ) - _raise_for_dbt_ls_failure(anchor_ls) - anchors_by_term[term] = tuple(node.unique_id for node in anchor_ls.nodes) - - selection: DbtInteropSelectionResult = resolve_dbt_interop_sqlbuild_selection( - project=project, - manifest=manifest, - graph=graph, - select=select, - exclude=exclude, - dbt_anchor_unique_ids_by_term=anchors_by_term, - ) - dbt_required_selector_terms: tuple[str, ...] = _build_required_dbt_selector_terms( - project=project, - manifest=manifest, - selected_model_names=selection.sqlbuild_model_names, - required_unique_ids=selection.dbt_required_unique_ids, - ) - supplemental_dbt_argvs: tuple[tuple[str, ...], ...] = _build_supplemental_dbt_argvs( - command=normalized_command, - dbt_executable=dbt_executable, - options=dbt_options, - selector_terms=dbt_required_selector_terms, - ) - return build_dbt_interop_plan( - command=normalized_command, - dbt_command_argv=_build_primary_dbt_argv( - command=normalized_command, - dbt_executable=dbt_executable, - dbt_command_args=dbt_command_args, - ), - dbt_ls_nodes=full_dbt_ls.nodes, - sqlbuild_command_argvs=_build_sqlbuild_argvs( - command=normalized_command, - sqlbuild_executable=sqlbuild_executable, - selected_model_names=selection.sqlbuild_model_names, - sqlbuild_command_args=sqlbuild_command_args, - ), - selection=selection, - dbt_required_selector_terms=dbt_required_selector_terms, - supplemental_dbt_command_argvs=supplemental_dbt_argvs, - ) - - -def _is_dbt_anchor_term(*, term: str, project: CompiledProjectView) -> bool: - parsed: SelectorExpansion = split_selector_expansion(term) - if not parsed.downstream: - return False - if DBT_PATH_SELECTOR_SEPARATOR in parsed.core: - return False - return not _matches_sqlbuild_direct_selector(term=parsed.core, project=project) - - -def _raise_for_dbt_ls_failure(result: DbtLsResult) -> None: - if result.command.returncode == 0: - return - raise DbtInteropRuntimeError("dbt ls failed", help=_dbt_failure_detail(result)) - - -def _dbt_failure_detail(result: DbtLsResult) -> str | None: - detail: str = (result.command.stderr or result.command.stdout).strip() - return detail or None - - -def _matches_sqlbuild_direct_selector(*, term: str, project: CompiledProjectView) -> bool: - model_names, _translated_path = resolve_sqlbuild_model_selector_names( - project=project, - term=term, - ) - return bool(model_names) - - -def _build_required_dbt_selector_terms( - *, - project: CompiledProjectView, - manifest: DbtManifestIndex, - selected_model_names: Sequence[str], - required_unique_ids: Sequence[str], -) -> tuple[str, ...]: - required_ids: frozenset[str] = frozenset(required_unique_ids) - if not required_ids: - return () - terms: set[str] = set() - for _model, dbt_model in resolve_sqlbuild_model_dbt_refs( - project=project, - manifest=manifest, - selected_model_names=selected_model_names, - ): - if dbt_model.unique_id not in required_ids: - continue - selector_term: str = dbt_fqn_selector_term( - fqn=dbt_model.fqn, - fallback=dbt_model.name, - ) - terms.add(f"+{selector_term}") - return tuple(sorted(terms)) - - -def _build_supplemental_dbt_argvs( - *, - command: DbtInteropCommand, - dbt_executable: str, - options: DbtCliOptions, - selector_terms: Sequence[str], -) -> tuple[tuple[str, ...], ...]: - if not selector_terms: - return () - dbt_command: str = "ls" if command == DbtInteropCommand.PLAN else command.value - argv: tuple[str, ...] = _append_dbt_options(argv=(dbt_executable, dbt_command), options=options) - return ((*argv, "--select", *selector_terms),) - - -def _build_primary_dbt_argv( - *, command: DbtInteropCommand, dbt_executable: str, dbt_command_args: Sequence[str] -) -> tuple[str, ...]: - dbt_command: str = "ls" if command == DbtInteropCommand.PLAN else command.value - return (dbt_executable, dbt_command, *dbt_command_args) - - -def _build_sqlbuild_argvs( - *, - command: DbtInteropCommand, - sqlbuild_executable: str, - selected_model_names: Sequence[str], - sqlbuild_command_args: Sequence[str], -) -> tuple[tuple[str, ...], ...]: - if not selected_model_names: - return () - sqlbuild_command: tuple[str, ...] = ( - ("build", "--no-tests", "--no-audits") - if command == DbtInteropCommand.RUN - else (command.value,) - ) - return ( - ( - sqlbuild_executable, - *sqlbuild_command, - "--select", - *selected_model_names, - *sqlbuild_command_args, - ), - ) - - -def _append_dbt_options(*, argv: tuple[str, ...], options: DbtCliOptions) -> tuple[str, ...]: - if options.project_dir is not None: - argv = (*argv, "--project-dir", str(options.project_dir)) - if options.profiles_dir is not None: - argv = (*argv, "--profiles-dir", str(options.profiles_dir)) - if options.target is not None: - argv = (*argv, "--target", options.target) - if options.target_path is not None: - argv = (*argv, "--target-path", str(options.target_path)) - if options.vars is not None: - argv = (*argv, "--vars", options.vars) - if options.state is not None: - argv = (*argv, "--state", str(options.state)) - if options.defer: - argv = (*argv, "--defer") - return argv diff --git a/src/sqlbuild/integrations/dbt/constants.py b/src/sqlbuild/integrations/dbt/constants.py index 71ad0da810..f092b0558b 100644 --- a/src/sqlbuild/integrations/dbt/constants.py +++ b/src/sqlbuild/integrations/dbt/constants.py @@ -23,6 +23,5 @@ ) DBT_EXECUTION_WARN_STATUSES: frozenset[str] = frozenset({"warn", "warning"}) DBT_FULL_REFRESH_FLAG: str = "--full-refresh" -DBT_PATH_SELECTOR_SEPARATOR: str = "~" DBT_SELECT_FLAG: str = "--select" DBT_UNIQUE_ID_SEPARATOR: str = "." diff --git a/src/sqlbuild/integrations/dbt/main/planning/__init__.py b/src/sqlbuild/integrations/dbt/main/planning/__init__.py deleted file mode 100644 index e8e98365d5..0000000000 --- a/src/sqlbuild/integrations/dbt/main/planning/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""dbt planning public operations.""" diff --git a/src/sqlbuild/integrations/dbt/main/planning/_plan_interop_command.py b/src/sqlbuild/integrations/dbt/main/planning/_plan_interop_command.py deleted file mode 100644 index aab36b7088..0000000000 --- a/src/sqlbuild/integrations/dbt/main/planning/_plan_interop_command.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Plan a dbt interop command.""" - -from sqlbuild._native import DbtCombinedGraph -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.integrations.dbt._helpers.planning.orchestration import ( - plan_dbt_interop_command as _plan, -) -from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner -from sqlbuild.integrations.dbt.models import ( - DbtCliOptions, - DbtInteropCommandArgs, - DbtInteropPlan, - DbtManifestIndex, -) -from sqlbuild.integrations.dbt.types import DbtInteropCommand - - -def plan_dbt_interop_command( - *, - command: DbtInteropCommand | str, - project: CompiledProjectView, - manifest: DbtManifestIndex, - graph: DbtCombinedGraph, - dbt_runner: DbtRunner, - dbt_options: DbtCliOptions, - command_args: DbtInteropCommandArgs, -) -> DbtInteropPlan: - """Plan dbt and SQLBuild work from compiled inputs.""" - - return _plan( - command=command, - project=project, - manifest=manifest, - graph=graph, - dbt_runner=dbt_runner, - dbt_options=dbt_options, - command_args=command_args, - ) diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py b/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py index 690374359e..47de125bf9 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py @@ -9,9 +9,9 @@ build_dbt_combined_graph, build_dbt_ls_argv, build_dbt_manifest_index, + plan_dbt_interop_command, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.integrations.dbt._helpers.planning.orchestration import plan_dbt_interop_command from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError from sqlbuild.integrations.dbt.models import ( From 45ed55489e38c9fa79e488f1fc1e8604b56993f3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:41:40 +0100 Subject: [PATCH 23/24] refactor: decide dbt execution argv, blocked models and summary natively --- .../sqlbuild-core/src/dbt_plan/constants.rs | 19 ++ .../dbt_plan/main/dbt_execution_ls_argv.rs | 23 +++ .../dbt_plan/main/dbt_failed_unique_ids.rs | 32 ++++ .../main/merged_dbt_execution_args.rs | 55 ++++++ crates/sqlbuild-core/src/dbt_plan/main/mod.rs | 4 + .../main/render_dbt_execution_summary.rs | 54 ++++++ crates/sqlbuild-core/src/dbt_plan/mod.rs | 1 + crates/sqlbuild-core/src/dbt_plan/models.rs | 9 + .../src/dbt_plan/tests/test_dbt_plan.rs | 77 +++++++- .../_helpers/dbt_interop/execution.rs | 119 +++++++++++++ .../bindings/_helpers/dbt_interop/graph.rs | 7 + .../src/bindings/_helpers/dbt_interop/mod.rs | 1 + .../_helpers/dbt_interop/registration.rs | 5 +- src/sqlbuild/_native.pyi | 17 ++ .../dbt/_helpers/pipeline/execute.py | 168 +----------------- .../dbt/_helpers/pipeline/execution_phases.py | 6 +- src/sqlbuild/integrations/dbt/constants.py | 7 - .../integrations/dbt/main/pipeline/execute.py | 6 +- .../sqlbuild/integrations/dbt/test_execute.py | 11 +- 19 files changed, 436 insertions(+), 185 deletions(-) create mode 100644 crates/sqlbuild-core/src/dbt_plan/constants.rs create mode 100644 crates/sqlbuild-core/src/dbt_plan/main/dbt_execution_ls_argv.rs create mode 100644 crates/sqlbuild-core/src/dbt_plan/main/dbt_failed_unique_ids.rs create mode 100644 crates/sqlbuild-core/src/dbt_plan/main/merged_dbt_execution_args.rs create mode 100644 crates/sqlbuild-core/src/dbt_plan/main/render_dbt_execution_summary.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs diff --git a/crates/sqlbuild-core/src/dbt_plan/constants.rs b/crates/sqlbuild-core/src/dbt_plan/constants.rs new file mode 100644 index 0000000000..beb16270d0 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_plan/constants.rs @@ -0,0 +1,19 @@ +//! dbt execution statuses and flags. + +pub const SUCCESS_STATUSES: [&str; 6] = ["ok", "success", "pass", "passed", "warn", "warning"]; +pub const WARN_STATUSES: [&str; 2] = ["warn", "warning"]; +pub const FAIL_STATUSES: [&str; 3] = ["error", "fail", "failed"]; +pub const SKIP_STATUSES: [&str; 2] = ["skip", "skipped"]; +/// Resolved option flags that take a value and are rebuilt from options. +pub const RESOLVED_VALUE_FLAGS: [&str; 6] = [ + "--project-dir", + "--profiles-dir", + "--target", + "--target-path", + "--vars", + "--state", +]; +pub const DEFER_FLAG: &str = "--defer"; +pub const SELECT_FLAG: &str = "--select"; +/// Execution flags `dbt ls` does not accept. +pub const LS_EXCLUDED_FLAGS: [&str; 2] = ["--full-refresh", "--fail-fast"]; diff --git a/crates/sqlbuild-core/src/dbt_plan/main/dbt_execution_ls_argv.rs b/crates/sqlbuild-core/src/dbt_plan/main/dbt_execution_ls_argv.rs new file mode 100644 index 0000000000..8ba6f3c50c --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_plan/main/dbt_execution_ls_argv.rs @@ -0,0 +1,23 @@ +//! The `dbt ls` command that counts what a dbt execution selects. + +use crate::dbt_plan::constants::LS_EXCLUDED_FLAGS; + +/// ` ls --output json` with the execution's arguments except flags `ls` rejects. +#[must_use] +pub fn dbt_execution_ls_argv(argv: &[String]) -> Option> { + let [executable, _command, args @ ..] = argv else { + return None; + }; + let mut converted: Vec = vec![ + executable.clone(), + "ls".to_owned(), + "--output".to_owned(), + "json".to_owned(), + ]; + converted.extend( + args.iter() + .filter(|arg| !LS_EXCLUDED_FLAGS.contains(&arg.as_str())) + .cloned(), + ); + Some(converted) +} diff --git a/crates/sqlbuild-core/src/dbt_plan/main/dbt_failed_unique_ids.rs b/crates/sqlbuild-core/src/dbt_plan/main/dbt_failed_unique_ids.rs new file mode 100644 index 0000000000..f777066858 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_plan/main/dbt_failed_unique_ids.rs @@ -0,0 +1,32 @@ +//! dbt models whose execution failed or was skipped. + +use std::collections::HashMap; + +use crate::dbt_plan::constants::SUCCESS_STATUSES; +use crate::dbt_plan::models::DbtExecutedNode; + +/// Unsuccessful models, plus the models unsuccessful tests and unit tests depend on. +#[must_use] +pub fn dbt_failed_unique_ids( + results: &[DbtExecutedNode], + validation_dependencies: &HashMap>, +) -> Vec { + let mut failed: Vec = Vec::new(); + for result in results { + if SUCCESS_STATUSES.contains(&result.status.to_lowercase().as_str()) { + continue; + } + match result.resource_type.as_str() { + "model" => failed.push(result.unique_id.clone()), + "test" | "unit_test" => failed.extend( + validation_dependencies + .get(&result.unique_id) + .into_iter() + .flatten() + .cloned(), + ), + _ => {} + } + } + failed +} diff --git a/crates/sqlbuild-core/src/dbt_plan/main/merged_dbt_execution_args.rs b/crates/sqlbuild-core/src/dbt_plan/main/merged_dbt_execution_args.rs new file mode 100644 index 0000000000..b3f6e13d97 --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_plan/main/merged_dbt_execution_args.rs @@ -0,0 +1,55 @@ +//! dbt execution arguments with SQLBuild's required dbt selectors merged in. + +use crate::dbt_plan::constants::{DEFER_FLAG, RESOLVED_VALUE_FLAGS, SELECT_FLAG}; + +fn without_resolved_options(args: &[String]) -> Vec { + let mut stripped: Vec = Vec::new(); + let mut index: usize = 0; + while index < args.len() { + let arg: &str = &args[index]; + if RESOLVED_VALUE_FLAGS.contains(&arg) { + index += 2; + continue; + } + if arg != DEFER_FLAG { + stripped.push(arg.to_owned()); + } + index += 1; + } + stripped +} + +/// Routed args without resolved options, every `--select` list extended by the missing terms. +#[must_use] +pub fn merged_dbt_execution_args(routed_args: &[String], extra_terms: &[String]) -> Vec { + let args: Vec = without_resolved_options(routed_args); + if extra_terms.is_empty() { + return args; + } + if !args.iter().any(|arg| arg == SELECT_FLAG) { + let mut merged: Vec = args; + merged.push(SELECT_FLAG.to_owned()); + merged.extend(extra_terms.iter().cloned()); + return merged; + } + let mut merged: Vec = Vec::new(); + let mut index: usize = 0; + while index < args.len() { + let token: &String = &args[index]; + merged.push(token.clone()); + index += 1; + if token != SELECT_FLAG { + continue; + } + while index < args.len() && !args[index].starts_with("--") { + merged.push(args[index].clone()); + index += 1; + } + for term in extra_terms { + if !merged.contains(term) { + merged.push(term.clone()); + } + } + } + merged +} diff --git a/crates/sqlbuild-core/src/dbt_plan/main/mod.rs b/crates/sqlbuild-core/src/dbt_plan/main/mod.rs index 9104e5cb9e..8946490fc4 100644 --- a/crates/sqlbuild-core/src/dbt_plan/main/mod.rs +++ b/crates/sqlbuild-core/src/dbt_plan/main/mod.rs @@ -1,4 +1,8 @@ pub mod dbt_anchor_term_check; +pub mod dbt_execution_ls_argv; +pub mod dbt_failed_unique_ids; pub mod dbt_failure_detail; pub mod dbt_plan_argvs; pub mod dbt_required_selector_terms; +pub mod merged_dbt_execution_args; +pub mod render_dbt_execution_summary; diff --git a/crates/sqlbuild-core/src/dbt_plan/main/render_dbt_execution_summary.rs b/crates/sqlbuild-core/src/dbt_plan/main/render_dbt_execution_summary.rs new file mode 100644 index 0000000000..6c7719b1ad --- /dev/null +++ b/crates/sqlbuild-core/src/dbt_plan/main/render_dbt_execution_summary.rs @@ -0,0 +1,54 @@ +//! The PASS/WARN/FAIL/SKIP/TOTAL footer of a dbt execution. + +use crate::cli_text::main::format_summary_footer::format_summary_footer; +use crate::cli_text::models::{CliStyle, SummaryCount}; +use crate::dbt_plan::constants::{FAIL_STATUSES, SKIP_STATUSES, SUCCESS_STATUSES, WARN_STATUSES}; +use crate::dbt_plan::models::DbtExecutedNode; + +/// The completion line and the counts with total elapsed time, or `None` without results. +#[must_use] +pub fn render_dbt_execution_summary( + results: &[DbtExecutedNode], + style: CliStyle, +) -> Option { + if results.is_empty() { + return None; + } + let (mut passed, mut warned, mut failed, mut skipped): (i64, i64, i64, i64) = (0, 0, 0, 0); + let mut elapsed: f64 = 0.0; + for result in results { + let status: String = result.status.to_lowercase(); + elapsed += result.execution_time.unwrap_or(0.0); + if WARN_STATUSES.contains(&status.as_str()) { + warned += 1; + } else if FAIL_STATUSES.contains(&status.as_str()) { + failed += 1; + } else if SKIP_STATUSES.contains(&status.as_str()) { + skipped += 1; + } else if SUCCESS_STATUSES.contains(&status.as_str()) { + passed += 1; + } + } + let status_line: String = if failed > 0 { + style.error("Completed with errors.") + } else if warned > 0 { + style.warning("Completed with warnings.") + } else { + style.success("Completed successfully.") + }; + let counts: Vec = [ + ("PASS", passed), + ("WARN", warned), + ("FAIL", failed), + ("SKIP", skipped), + ("TOTAL", passed + warned + failed + skipped), + ] + .into_iter() + .map(|(label, value)| SummaryCount { + label: label.to_owned(), + value, + }) + .collect(); + let footer: String = format_summary_footer(style, &counts, Some(&format!("{elapsed:.2}s"))); + Some(format!("{status_line}\n{footer}")) +} diff --git a/crates/sqlbuild-core/src/dbt_plan/mod.rs b/crates/sqlbuild-core/src/dbt_plan/mod.rs index 7452d2ad1b..271470641a 100644 --- a/crates/sqlbuild-core/src/dbt_plan/mod.rs +++ b/crates/sqlbuild-core/src/dbt_plan/mod.rs @@ -1,5 +1,6 @@ //! `sqb dbt` plan and execution decisions: command lines, required dbt selectors and anchors. +pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-core/src/dbt_plan/models.rs b/crates/sqlbuild-core/src/dbt_plan/models.rs index 74d6a6c9d0..80547388f2 100644 --- a/crates/sqlbuild-core/src/dbt_plan/models.rs +++ b/crates/sqlbuild-core/src/dbt_plan/models.rs @@ -30,3 +30,12 @@ pub struct DbtReferencedModel { pub fqn: Vec, pub name: String, } + +/// One executed dbt node's outcome. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DbtExecutedNode { + pub unique_id: String, + pub resource_type: String, + pub status: String, + pub execution_time: Option, +} diff --git a/crates/sqlbuild-core/src/dbt_plan/tests/test_dbt_plan.rs b/crates/sqlbuild-core/src/dbt_plan/tests/test_dbt_plan.rs index 0f336a250d..00aab724b1 100644 --- a/crates/sqlbuild-core/src/dbt_plan/tests/test_dbt_plan.rs +++ b/crates/sqlbuild-core/src/dbt_plan/tests/test_dbt_plan.rs @@ -1,10 +1,14 @@ use crate::dbt_cli::models::{DbtCliOptionValues, DbtInteropCommandKind}; +use crate::dbt_plan::main::dbt_execution_ls_argv::dbt_execution_ls_argv; +use crate::dbt_plan::main::dbt_failed_unique_ids::dbt_failed_unique_ids; use crate::dbt_plan::main::dbt_failure_detail::dbt_failure_detail; use crate::dbt_plan::main::dbt_plan_argvs::dbt_plan_argvs; use crate::dbt_plan::main::dbt_required_selector_terms::dbt_required_selector_terms; -use crate::dbt_plan::models::{DbtPlanArgvRequest, DbtReferencedModel}; +use crate::dbt_plan::main::merged_dbt_execution_args::merged_dbt_execution_args; +use crate::dbt_plan::models::{DbtExecutedNode, DbtPlanArgvRequest, DbtReferencedModel}; use crate::dbt_plan::tests::helpers::{flattened, texts}; use crate::dbt_plan::tests::test_types::{ArgvsTestCase, TextsTestCase}; +use std::collections::HashMap; #[test] fn given_plan_inputs_when_building_argvs_then_maps_commands() { @@ -146,3 +150,74 @@ fn given_dbt_outputs_when_summarising_then_returns_terms_and_details() { assert_eq!(texts, test_case.expected_texts, "{}", test_case.description); } } + +#[test] +fn given_execution_inputs_when_deciding_then_merges_counts_and_blocks() { + let failed_test: DbtExecutedNode = DbtExecutedNode { + unique_id: "test.a.t".to_owned(), + resource_type: "test".to_owned(), + status: "FAIL".to_owned(), + execution_time: None, + }; + let test_cases = [ + ArgvsTestCase { + description: "resolved options are dropped and missing terms join each selection", + argvs: vec![ + merged_dbt_execution_args( + &texts(&[ + "--target", + "t", + "--select", + "a", + "+b", + "--defer", + "--threads", + "4", + ]), + &texts(&["+b", "+c"]), + ), + merged_dbt_execution_args(&texts(&["--vars"]), &texts(&["+c"])), + ], + expected_argvs: vec![ + vec!["--select", "a", "+b", "+c", "--threads", "4"], + vec!["--select", "+c"], + ], + }, + ArgvsTestCase { + description: "the counting ls drops flags ls rejects", + argvs: vec![ + dbt_execution_ls_argv(&texts(&["dbt", "build", "--full-refresh", "--select", "a"])) + .unwrap_or_default(), + dbt_execution_ls_argv(&texts(&["dbt"])).unwrap_or_default(), + ], + expected_argvs: vec![ + vec!["dbt", "ls", "--output", "json", "--select", "a"], + vec![], + ], + }, + ArgvsTestCase { + description: "failed tests block the models they check", + argvs: vec![dbt_failed_unique_ids( + &[ + failed_test.clone(), + DbtExecutedNode { + unique_id: "model.a.ok".to_owned(), + resource_type: "model".to_owned(), + status: "Success".to_owned(), + execution_time: Some(1.0), + }, + ], + &HashMap::from([("test.a.t".to_owned(), texts(&["model.a.m"]))]), + )], + expected_argvs: vec![vec!["model.a.m"]], + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_case.argvs, test_case.expected_argvs, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs new file mode 100644 index 0000000000..2c4559cdbe --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs @@ -0,0 +1,119 @@ +//! dbt execution decisions: merged argv, the counting `dbt ls`, failed models and the footer. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::dbt_cli::main::build_dbt_argv::build_dbt_argv; +use sqlbuild_core::dbt_cli::models::DbtArgvCommand; +use sqlbuild_core::dbt_graph::main::dbt_downstream_sqlbuild_models::dbt_downstream_sqlbuild_models; +use sqlbuild_core::dbt_plan::main::dbt_execution_ls_argv::dbt_execution_ls_argv as ls_argv; +use sqlbuild_core::dbt_plan::main::dbt_failed_unique_ids::dbt_failed_unique_ids; +use sqlbuild_core::dbt_plan::main::merged_dbt_execution_args::merged_dbt_execution_args; +use sqlbuild_core::dbt_plan::main::render_dbt_execution_summary::render_dbt_execution_summary; +use sqlbuild_core::dbt_plan::models::DbtExecutedNode; + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::dbt_interop::cli::option_values; +use crate::bindings::_helpers::dbt_interop::graph::{DbtCombinedGraph, texts}; + +const MERGED_OPTIONS: (&[&str], &[&str]) = (&["command", "options", "routed_args", "plan"], &[]); + +/// Merge the original dbt selection with dbt models SQLBuild work requires; `None` without dbt work. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_merged_dbt_execution_argv<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult>> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_merged_dbt_execution_argv", + options, + MERGED_OPTIONS, + )?; + let plan: Bound<'py, PyAny> = arguments.value("plan")?; + let required_terms: Vec = texts(&plan.getattr("dbt_required_selector_terms")?)?; + if !plan.getattr("dbt_selected_unique_ids")?.is_truthy()? && required_terms.is_empty() { + return Ok(None); + } + let args: Vec = + merged_dbt_execution_args(&texts(&arguments.value("routed_args")?)?, &required_terms); + let dbt_executable: String = plan.getattr("dbt_command_argv")?.get_item(0)?.extract()?; + let command: DbtArgvCommand = DbtArgvCommand::Other { + command: arguments.value("command")?.getattr("value")?.extract()?, + args, + }; + let options = option_values(&arguments.value("options")?)?; + Ok(Some(PyTuple::new( + py, + build_dbt_argv(&dbt_executable, &options, command), + )?)) +} + +/// The `dbt ls --output json` that counts an execution argv's selection, when it has a command. +#[pyfunction] +#[pyo3(signature = (*, argv))] +fn dbt_execution_ls_argv(argv: Vec) -> Option> { + ls_argv(&argv) +} + +fn executed_nodes(node_results: &Bound<'_, PyAny>) -> PyResult> { + let mut nodes: Vec = Vec::new(); + for result in node_results.try_iter()? { + let result: Bound<'_, PyAny> = result?; + nodes.push(DbtExecutedNode { + unique_id: result.getattr("unique_id")?.extract()?, + resource_type: result.getattr("resource_type")?.extract()?, + status: result.getattr("status")?.extract()?, + execution_time: result.getattr("execution_time")?.extract()?, + }); + } + Ok(nodes) +} + +/// SQLBuild models downstream of failed or skipped dbt models and the models failed tests check. +#[pyfunction] +#[pyo3(signature = (*, graph, manifest, node_results))] +fn build_failed_sqlbuild_model_names<'py>( + graph: &Bound<'py, DbtCombinedGraph>, + manifest: &Bound<'py, PyAny>, + node_results: &Bound<'py, PyAny>, +) -> PyResult> { + let mut dependencies: HashMap> = HashMap::new(); + let lookup: Bound<'py, PyAny> = manifest.getattr("validation_depends_on_nodes_by_unique_id")?; + for item in lookup.call_method0("items")?.try_iter()? { + let (unique_id, depends_on): (String, Bound<'py, PyAny>) = item?.extract()?; + dependencies.insert(unique_id, texts(&depends_on)?); + } + let failed: Vec = dbt_failed_unique_ids(&executed_nodes(node_results)?, &dependencies); + PyTuple::new( + graph.py(), + dbt_downstream_sqlbuild_models(graph.get().native(), &failed), + ) +} + +/// Render a PASS/WARN/FAIL/SKIP/TOTAL footer for executed dbt nodes. +#[pyfunction] +#[pyo3(signature = (*, node_results, use_color))] +fn render_dbt_execution_summary_footer( + node_results: &Bound<'_, PyAny>, + use_color: bool, +) -> PyResult> { + Ok(render_dbt_execution_summary( + &executed_nodes(node_results)?, + CliStyle { use_color }, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_merged_dbt_execution_argv, module)?)?; + module.add_function(wrap_pyfunction!(dbt_execution_ls_argv, module)?)?; + module.add_function(wrap_pyfunction!(build_failed_sqlbuild_model_names, module)?)?; + module.add_function(wrap_pyfunction!( + render_dbt_execution_summary_footer, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs index 681478de22..89c30819de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs @@ -57,6 +57,13 @@ fn stable_edges<'py>(py: Python<'py>, edges: &DbtGraphEdges) -> PyResult &CombinedGraph { + &self.graph + } +} + #[pymethods] impl DbtCombinedGraph { /// Every node's stable ID. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs index 06322ff9e8..7fde8df5ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod cli; +pub(crate) mod execution; pub(crate) mod graph; pub(crate) mod manifest; pub(crate) mod orchestration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs index 0aed90bc7f..edbec1eca6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs @@ -2,10 +2,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::dbt_interop::{cli, graph, manifest, orchestration, plan, profile}; +use crate::bindings::_helpers::dbt_interop::{ + cli, execution, graph, manifest, orchestration, plan, profile, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { cli::register(module)?; + execution::register(module)?; graph::register(module)?; manifest::register(module)?; orchestration::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index a6a8e948f5..bb9aee70a3 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1492,6 +1492,23 @@ def resolve_dbt_interop_sqlbuild_selection( dbt_anchor_unique_ids_by_term: Mapping[str, Sequence[str]] | None = None, ) -> DbtInteropSelectionResult: ... def dbt_fqn_selector_term(*, fqn: Sequence[str], fallback: str) -> str: ... +def build_merged_dbt_execution_argv( + *, + command: DbtInteropCommand, + options: DbtCliOptions, + routed_args: Sequence[str], + plan: DbtInteropPlan, +) -> tuple[str, ...] | None: ... +def dbt_execution_ls_argv(*, argv: Sequence[str]) -> list[str] | None: ... +def build_failed_sqlbuild_model_names( + *, + graph: DbtCombinedGraph, + manifest: DbtManifestIndex, + node_results: Sequence[DbtNodeExecutionResult], +) -> tuple[str, ...]: ... +def render_dbt_execution_summary_footer( + *, node_results: Sequence[DbtNodeExecutionResult], use_color: bool +) -> str | None: ... def plan_dbt_interop_command( *, command: DbtInteropCommand | str, diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py index 30bd9b185f..70ac8c252b 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execute.py @@ -9,22 +9,12 @@ from typing import TextIO, cast from sqlbuild._native import ( - DbtCombinedGraph, - build_dbt_command_argv, - format_summary_footer, + dbt_execution_ls_argv, parse_dbt_ls_json_lines, render_dbt_node_result, ) from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner -from sqlbuild.integrations.dbt.constants import ( - DBT_DEFER_FLAG, - DBT_EXECUTION_FAIL_STATUSES, - DBT_EXECUTION_SKIP_STATUSES, - DBT_EXECUTION_SUCCESS_STATUSES, - DBT_EXECUTION_WARN_STATUSES, - DBT_SELECT_FLAG, - DBT_UNIQUE_ID_SEPARATOR, -) +from sqlbuild.integrations.dbt.constants import DBT_UNIQUE_ID_SEPARATOR from sqlbuild.integrations.dbt.main.runtime._execute_json_event_stream import ( execute_dbt_json_event_stream, ) @@ -32,18 +22,14 @@ DbtCliOptions, DbtCommandExecutionResult, DbtCommandResult, - DbtInteropPlan, DbtLsNode, - DbtManifestIndex, DbtNodeExecutionResult, DbtNodeMessage, ) -from sqlbuild.integrations.dbt.types import DbtInteropCommand, DbtSupportedResourceType from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter _COMMAND_AND_SUBCOMMAND_ARGUMENT_COUNT: int = 2 -_DBT_LS_EXCLUDED_EXECUTION_FLAGS: frozenset[str] = frozenset({"--full-refresh", "--fail-fast"}) def execute_dbt_commands( @@ -135,100 +121,6 @@ def execute_dbt_commands( return DbtCommandExecutionResult(returncode=returncode, node_results=results) -def build_merged_dbt_execution_argv( - *, - command: DbtInteropCommand, - options: DbtCliOptions, - routed_args: tuple[str, ...], - plan: DbtInteropPlan, -) -> tuple[str, ...] | None: - """Merge original dbt selection with dbt models required by SQLBuild work.""" - - required_terms: tuple[str, ...] = plan.dbt_required_selector_terms - if not plan.dbt_selected_unique_ids and not required_terms: - return None - merged_args: tuple[str, ...] = _merge_dbt_select_terms( - args=_strip_resolved_dbt_options(routed_args), - extra_terms=required_terms, - ) - return build_dbt_command_argv( - dbt_executable=plan.dbt_command_argv[0], - command=command.value, - options=options, - args=merged_args, - ) - - -def build_failed_sqlbuild_model_names( - *, - graph: DbtCombinedGraph, - manifest: DbtManifestIndex, - node_results: tuple[DbtNodeExecutionResult, ...], -) -> tuple[str, ...]: - """Return SQLBuild models downstream of actual failed or skipped dbt work.""" - - failed_unique_ids: set[str] = set() - for result in node_results: - if result.status.lower() in DBT_EXECUTION_SUCCESS_STATUSES: - continue - if result.resource_type == DbtSupportedResourceType.MODEL: - failed_unique_ids.add(result.unique_id) - elif result.resource_type in ( - DbtSupportedResourceType.TEST, - DbtSupportedResourceType.UNIT_TEST, - ): - failed_unique_ids.update( - manifest.validation_depends_on_nodes_by_unique_id.get(result.unique_id, ()) - ) - return graph.downstream_sqlbuild_model_names(failed_unique_ids) - - -def render_dbt_execution_summary_footer( - *, node_results: tuple[DbtNodeExecutionResult, ...], use_color: bool -) -> str | None: - """Render a PASS/WARN/FAIL/SKIP/TOTAL footer for executed dbt nodes.""" - - if not node_results: - return None - pass_count: int = 0 - warn_count: int = 0 - fail_count: int = 0 - skip_count: int = 0 - elapsed: float = 0.0 - for result in node_results: - status: str = result.status.lower() - if result.execution_time is not None: - elapsed += result.execution_time - if status in DBT_EXECUTION_WARN_STATUSES: - warn_count += 1 - elif status in DBT_EXECUTION_FAIL_STATUSES: - fail_count += 1 - elif status in DBT_EXECUTION_SKIP_STATUSES: - skip_count += 1 - elif status in DBT_EXECUTION_SUCCESS_STATUSES: - pass_count += 1 - total_count: int = pass_count + warn_count + fail_count + skip_count - style: CliStyle = CliStyle(use_color=use_color) - if fail_count > 0: - status_line: str = style.error("Completed with errors.") - elif warn_count > 0: - status_line = style.warning("Completed with warnings.") - else: - status_line = style.success("Completed successfully.") - counts_line: str = format_summary_footer( - counts=( - ("PASS", pass_count), - ("WARN", warn_count), - ("FAIL", fail_count), - ("SKIP", skip_count), - ("TOTAL", total_count), - ), - use_color=use_color, - elapsed=f"{elapsed:.2f}s", - ) - return f"{status_line}\n{counts_line}" - - def _dbt_execution_expected_total( *, runner: DbtRunner, @@ -237,14 +129,14 @@ def _dbt_execution_expected_total( stream: TextIO, use_color: bool, ) -> int | None: - ls_argv: tuple[str, ...] | None = _dbt_ls_argv_from_execution_argv(argv) + ls_argv: list[str] | None = dbt_execution_ls_argv(argv=argv) if ls_argv is None: return None status: TransientStatusReporter = TransientStatusReporter(stream=stream, use_color=use_color) status.start("Resolving dbt execution selection...") start: float = time.monotonic() try: - result: DbtCommandResult = runner.invoke(argv=ls_argv, cwd=options.project_dir) + result: DbtCommandResult = runner.invoke(argv=tuple(ls_argv), cwd=options.project_dir) finally: status.complete( message=f"Resolved dbt execution selection. ({time.monotonic() - start:.2f}s)" @@ -254,17 +146,6 @@ def _dbt_execution_expected_total( return len(parse_dbt_ls_json_lines(stdout=result.stdout)) -def _dbt_ls_argv_from_execution_argv(argv: tuple[str, ...]) -> tuple[str, ...] | None: - if len(argv) < _COMMAND_AND_SUBCOMMAND_ARGUMENT_COUNT: - return None - converted: list[str] = [argv[0], "ls", "--output", "json"] - for arg in argv[2:]: - if arg in _DBT_LS_EXCLUDED_EXECUTION_FLAGS: - continue - converted.append(arg) - return tuple(converted) - - def _append_missing_run_results( *, results: tuple[DbtNodeExecutionResult, ...], @@ -343,44 +224,3 @@ def _messages_from_run_result(raw_result: dict[str, object]) -> tuple[DbtNodeMes if isinstance(failures, int) and failures > 0: return (DbtNodeMessage(level="error", message=f"{failures} failure(s)"),) return () - - -def _strip_resolved_dbt_options(args: tuple[str, ...]) -> tuple[str, ...]: - value_flags: frozenset[str] = frozenset( - {"--project-dir", "--profiles-dir", "--target", "--target-path", "--vars", "--state"} - ) - stripped: list[str] = [] - index: int = 0 - while index < len(args): - arg: str = args[index] - if arg in value_flags: - index += 2 - continue - if arg == DBT_DEFER_FLAG: - index += 1 - continue - stripped.append(arg) - index += 1 - return tuple(stripped) - - -def _merge_dbt_select_terms( - *, args: tuple[str, ...], extra_terms: tuple[str, ...] -) -> tuple[str, ...]: - if not extra_terms: - return args - if DBT_SELECT_FLAG not in args: - return (*args, DBT_SELECT_FLAG, *extra_terms) - merged: list[str] = [] - index: int = 0 - while index < len(args): - token: str = args[index] - merged.append(token) - index += 1 - if token != DBT_SELECT_FLAG: - continue - while index < len(args) and not args[index].startswith("--"): - merged.append(args[index]) - index += 1 - merged.extend(term for term in extra_terms if term not in merged) - return tuple(merged) diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py index 1df7b8fb1f..859ca77a78 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( ConnectionProgressReporter, apply_command_warehouse, + render_dbt_execution_summary_footer, render_dbt_interop_plan, ) from sqlbuild.cli.commands.main.dbt.dbt_sqlbuild_work import execute_dbt_sqlbuild_work @@ -17,10 +18,7 @@ from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.integrations.dbt._helpers.pipeline.execute import ( - execute_dbt_commands, - render_dbt_execution_summary_footer, -) +from sqlbuild.integrations.dbt._helpers.pipeline.execute import execute_dbt_commands from sqlbuild.integrations.dbt._helpers.pipeline.plan_output import build_sqlbuild_plan_output from sqlbuild.integrations.dbt.main.profile._resolve_connection_config import ( resolve_connection_config, diff --git a/src/sqlbuild/integrations/dbt/constants.py b/src/sqlbuild/integrations/dbt/constants.py index f092b0558b..e6b2f3d9d6 100644 --- a/src/sqlbuild/integrations/dbt/constants.py +++ b/src/sqlbuild/integrations/dbt/constants.py @@ -16,12 +16,5 @@ DBT_DEFER_FLAG: str = "--defer" -DBT_EXECUTION_FAIL_STATUSES: frozenset[str] = frozenset({"error", "fail", "failed"}) -DBT_EXECUTION_SKIP_STATUSES: frozenset[str] = frozenset({"skip", "skipped"}) -DBT_EXECUTION_SUCCESS_STATUSES: frozenset[str] = frozenset( - {"ok", "success", "pass", "passed", "warn", "warning"} -) -DBT_EXECUTION_WARN_STATUSES: frozenset[str] = frozenset({"warn", "warning"}) DBT_FULL_REFRESH_FLAG: str = "--full-refresh" -DBT_SELECT_FLAG: str = "--select" DBT_UNIQUE_ID_SEPARATOR: str = "." diff --git a/src/sqlbuild/integrations/dbt/main/pipeline/execute.py b/src/sqlbuild/integrations/dbt/main/pipeline/execute.py index fd63739f65..cb4375e9ef 100644 --- a/src/sqlbuild/integrations/dbt/main/pipeline/execute.py +++ b/src/sqlbuild/integrations/dbt/main/pipeline/execute.py @@ -2,12 +2,12 @@ from __future__ import annotations -from sqlbuild._native import plan_has_executable_work -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.integrations.dbt._helpers.pipeline.execute import ( +from sqlbuild._native import ( build_failed_sqlbuild_model_names, build_merged_dbt_execution_argv, + plan_has_executable_work, ) +from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.integrations.dbt._helpers.pipeline.execution_phases import ( execute_dbt_without_state_tracking, resolve_dbt_connection_config, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py index 1b00aa9140..61109771df 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py @@ -7,16 +7,17 @@ import pytest -from sqlbuild._native import DbtCombinedGraph, build_dbt_interop_plan +from sqlbuild._native import ( + DbtCombinedGraph, + build_dbt_interop_plan, + build_failed_sqlbuild_model_names, + build_merged_dbt_execution_argv, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt._helpers.pipeline import execute as dbt_execute_module from sqlbuild.integrations.dbt._helpers.pipeline import execution_phases as phases_module -from sqlbuild.integrations.dbt._helpers.pipeline.execute import ( - build_failed_sqlbuild_model_names, - build_merged_dbt_execution_argv, -) from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.main.pipeline import execute as execute_module from sqlbuild.integrations.dbt.main.pipeline import plan as plan_module From 6c26d5529748fd62c2bf259bb7dfe3514234baa6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:50:10 +0100 Subject: [PATCH 24/24] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 16ad19959b..e14c07366d 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 63635 +remaining_lines = 62087 [[stays]] path = "executor"