From 195a1a78e933b83da1bb15ff6bab25065d4e10f3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:09:37 +0100 Subject: [PATCH 01/11] refactor: render SQL test comparisons and resource ids natively for compile and CLI callers Rust compile, plan and CLI paths no longer round-trip through executor Python to render SQL test comparison and difference-sample SQL, describe missing expected columns, build scenario comparison SQL or format SQL test and audit resource ids. Planner, rules and refactor type comparisons call the native type normaliser directly instead of the Python types_equal wrapper. --- .../src/compiler/constants.rs | 4 + .../compiler/main/missing_expected_columns.rs | 43 ++++ .../src/compiler/main/mod.rs | 2 + .../main/scenario_expected_comparison_sql.rs | 17 ++ .../src/nested_progress/main/mod.rs | 1 + .../main/sql_test_resource_id.rs | 12 + .../src/bindings/_helpers/analysis/mod.rs | 1 + .../_helpers/analysis/sql_test_comparisons.rs | 242 ++++++++++++++++++ .../bindings/_helpers/analysis/sql_tests.rs | 23 -- .../_helpers/compile_command/target_writer.rs | 18 +- .../compile_outputs/runtime_target.rs | 21 +- .../_helpers/planner_changes/detect.rs | 16 +- .../_helpers/planner_resolution/sources.rs | 21 +- .../_helpers/refactor/contract_compare.rs | 17 +- .../bindings/_helpers/result_output/mod.rs | 1 + .../_helpers/result_output/projection.rs | 32 +-- .../_helpers/result_output/registration.rs | 3 +- .../_helpers/result_output/resource_ids.rs | 104 ++++++++ .../src/bindings/_helpers/rules/rows.rs | 27 +- .../_helpers/scenario_command/conversions.rs | 29 +-- .../_helpers/sql_test_command/test_command.rs | 12 +- .../_helpers/sql_test_glue/plan_entries.rs | 26 +- .../bindings/_helpers/type_system/equality.rs | 90 +++++++ .../src/bindings/_helpers/type_system/mod.rs | 1 + .../src/bindings/main/register.rs | 1 + src/sqlbuild/_native.pyi | 36 +++ .../executor/auditing/main/resource_id.py | 28 -- .../executor/build/_helpers/end_audits.py | 2 +- .../executor/build/_helpers/source_audits.py | 2 +- .../executor/build/classes/build_scheduler.py | 3 +- .../executor/pipeline/_helpers/auditing.py | 3 +- .../executor/pipeline/_helpers/testing.py | 2 +- .../scenario/main/_expected_expectations.py | 4 +- .../scenario/main/expected_comparison_sql.py | 24 -- .../testing/_helpers/difference_samples.py | 50 +--- .../testing/_helpers/expected_column_probe.py | 26 +- .../testing/_helpers/native_requests.py | 49 ---- src/sqlbuild/executor/testing/exceptions.py | 5 - .../executor/testing/main/_execute.py | 6 +- .../testing/main/_missing_expected_columns.py | 34 --- .../executor/testing/main/_resource_id.py | 17 -- .../executor/testing/main/comparison_sql.py | 24 -- .../testing/main/comparison_sql_batch.py | 54 ---- src/sqlbuild/executor/testing/types.py | 9 - .../src/sqlbuild/executor/testing/helpers.py | 8 +- .../testing/test_sql_test_build_parity.py | 6 +- .../testing/test_sql_test_execution.py | 32 ++- .../_helpers/test_module_paths.py | 1 + .../compile/test_compile_target_writer.py | 4 +- .../sqlbuild/cli/output/_helpers/helpers.py | 2 +- .../_helpers/sql_test_assembly/helpers.py | 8 +- .../sql_test_assembly/test_plan_test.py | 14 +- .../pipeline/_helpers/test_auditing.py | 3 +- .../main/test_expected_comparison_sql.py | 4 +- .../testing/main/test_comparison_sql.py | 87 ++++--- 55 files changed, 708 insertions(+), 603 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/compiler/main/missing_expected_columns.rs create mode 100644 crates/sqlbuild-analysis/src/compiler/main/scenario_expected_comparison_sql.rs create mode 100644 crates/sqlbuild-core/src/nested_progress/main/sql_test_resource_id.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/type_system/equality.rs delete mode 100644 src/sqlbuild/executor/auditing/main/resource_id.py delete mode 100644 src/sqlbuild/executor/scenario/main/expected_comparison_sql.py delete mode 100644 src/sqlbuild/executor/testing/_helpers/native_requests.py delete mode 100644 src/sqlbuild/executor/testing/exceptions.py delete mode 100644 src/sqlbuild/executor/testing/main/_missing_expected_columns.py delete mode 100644 src/sqlbuild/executor/testing/main/_resource_id.py delete mode 100644 src/sqlbuild/executor/testing/main/comparison_sql.py delete mode 100644 src/sqlbuild/executor/testing/main/comparison_sql_batch.py diff --git a/crates/sqlbuild-analysis/src/compiler/constants.rs b/crates/sqlbuild-analysis/src/compiler/constants.rs index 179f2ae1b7..6ad99688fa 100644 --- a/crates/sqlbuild-analysis/src/compiler/constants.rs +++ b/crates/sqlbuild-analysis/src/compiler/constants.rs @@ -13,3 +13,7 @@ pub(crate) const FUNCTION_TYPE_DIALECTS: [(&str, &str, DialectType); 4] = [ pub(crate) const CTES_FIELD: &str = "ctes"; pub(crate) const ALIAS_FIELD: &str = "alias"; pub(crate) const NAME_FIELD: &str = "name"; + +/// Opening and closing quotes of identifiers SQL tests list as expected columns. +pub(crate) const SQL_TEST_IDENTIFIER_QUOTES: [(char, char); 3] = + [('"', '"'), ('`', '`'), ('[', ']')]; diff --git a/crates/sqlbuild-analysis/src/compiler/main/missing_expected_columns.rs b/crates/sqlbuild-analysis/src/compiler/main/missing_expected_columns.rs new file mode 100644 index 0000000000..b3a0829cb2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/main/missing_expected_columns.rs @@ -0,0 +1,43 @@ +//! Explain expected-output columns that a model does not produce. + +use sqlbuild_core::text::main::python_casefold::python_casefold; + +use crate::compiler::constants::SQL_TEST_IDENTIFIER_QUOTES; + +/// The message for listed expected columns absent from casefolded model output names, if any. +#[must_use] +pub fn missing_expected_columns( + model_name: &str, + expected_columns: &[String], + available_columns: &[String], +) -> Option { + let missing: Vec<&str> = expected_columns + .iter() + .filter(|column| !available_columns.contains(&normalized_identifier(column))) + .map(String::as_str) + .collect(); + if missing.is_empty() { + return None; + } + Some(format!( + "expected output '{model_name}' lists columns that model '{model_name}' does not output: {}. Expected CTEs are compared only on the columns they list, matched by name", + missing.join(", ") + )) +} + +fn normalized_identifier(identifier: &str) -> String { + let closing: Option = identifier.chars().next().and_then(|first| { + SQL_TEST_IDENTIFIER_QUOTES + .iter() + .find(|(open, _)| *open == first) + .map(|(_, close)| *close) + }); + match closing { + Some(close) if identifier.ends_with(close) => { + let inner: &str = identifier.get(1..identifier.len() - 1).unwrap_or_default(); + let doubled: String = [close, close].iter().collect(); + python_casefold(&inner.replace(&doubled, &close.to_string())) + } + _ => python_casefold(identifier), + } +} diff --git a/crates/sqlbuild-analysis/src/compiler/main/mod.rs b/crates/sqlbuild-analysis/src/compiler/main/mod.rs index 0975b294d5..e33178c38d 100644 --- a/crates/sqlbuild-analysis/src/compiler/main/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/main/mod.rs @@ -1,4 +1,6 @@ pub mod function_type_validation; +pub mod missing_expected_columns; +pub mod scenario_expected_comparison_sql; pub mod sql_scenario_classification; pub mod sql_test_assembly; pub mod sql_test_chains; diff --git a/crates/sqlbuild-analysis/src/compiler/main/scenario_expected_comparison_sql.rs b/crates/sqlbuild-analysis/src/compiler/main/scenario_expected_comparison_sql.rs new file mode 100644 index 0000000000..3ccdd0b378 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/main/scenario_expected_comparison_sql.rs @@ -0,0 +1,17 @@ +//! The query comparing a scenario's actual output with its expected SQL. + +/// Actual, expected and mismatched row counts of one scenario expectation. +#[must_use] +pub fn scenario_expected_comparison_sql( + actual_sql: &str, + expected_sql: &str, + set_difference_operator: &str, +) -> String { + format!( + "WITH __actual AS ({actual_sql}), __expected AS ({expected_sql}) SELECT \ + (SELECT COUNT(*) FROM __actual) AS actual_count, \ + (SELECT COUNT(*) FROM __expected) AS expected_count, \ + (SELECT COUNT(*) FROM (SELECT * FROM __actual {set_difference_operator} SELECT * FROM __expected\ + ) AS __sqlbuild_mismatch) AS mismatched_count" + ) +} diff --git a/crates/sqlbuild-core/src/nested_progress/main/mod.rs b/crates/sqlbuild-core/src/nested_progress/main/mod.rs index 1f0a78ad20..b42103fe36 100644 --- a/crates/sqlbuild-core/src/nested_progress/main/mod.rs +++ b/crates/sqlbuild-core/src/nested_progress/main/mod.rs @@ -5,3 +5,4 @@ pub mod render_audit_row; pub mod render_group_header; pub mod render_nested_item; pub mod render_nested_spinner_line; +pub mod sql_test_resource_id; diff --git a/crates/sqlbuild-core/src/nested_progress/main/sql_test_resource_id.rs b/crates/sqlbuild-core/src/nested_progress/main/sql_test_resource_id.rs new file mode 100644 index 0000000000..8d68a7cc68 --- /dev/null +++ b/crates/sqlbuild-core/src/nested_progress/main/sql_test_resource_id.rs @@ -0,0 +1,12 @@ +//! The execution-protocol identity of one SQL test case. + +/// `sql_test:`, or `sql_test:::` for one parameterised case. +#[must_use] +pub fn sql_test_resource_id(test_name: &str, case: Option<(&str, i64, &str)>) -> String { + match case { + Some((source_path, block_index, case_name)) => { + format!("sql_test:{source_path}:{block_index}:{case_name}") + } + None => format!("sql_test:{test_name}"), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs index 1e4d77872b..d13e5033ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs @@ -11,4 +11,5 @@ pub(crate) mod reference_facts; pub(crate) mod relation_names; mod required_columns; pub(crate) mod schema_request; +pub(crate) mod sql_test_comparisons; pub(crate) mod sql_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs new file mode 100644 index 0000000000..c45df16436 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs @@ -0,0 +1,242 @@ +//! Render executable SQL test and scenario comparison queries from planned entries. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use serde_json::{Value, json}; +use sqlbuild_analysis::compiler::main::missing_expected_columns::missing_expected_columns; +use sqlbuild_analysis::compiler::main::scenario_expected_comparison_sql::scenario_expected_comparison_sql; +use sqlbuild_analysis::compiler::main::sql_test_difference_sampling::render_difference_sample_json; +use sqlbuild_analysis::compiler::main::sql_test_rendering::render_json; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::types::CompilerDetach; + +const RENDER_WORKERS: usize = 4; +const DEFAULT_SET_DIFFERENCE: &str = "EXCEPT"; +const TOP_CLAUSE_DIALECT: &str = "tsql"; + +/// Operator, dialect and optional probed chain step of one comparison render. +pub(crate) struct RenderSettings { + set_difference_operator: String, + sql_analysis_dialect: Option, + probe_step_index: Option, +} + +impl RenderSettings { + fn read(options: Option<&Bound<'_, PyDict>>) -> PyResult { + Ok(Self { + set_difference_operator: optional_value(options, "set_difference_operator")? + .unwrap_or_else(|| DEFAULT_SET_DIFFERENCE.to_owned()), + sql_analysis_dialect: optional_value(options, "sql_analysis_dialect")?, + probe_step_index: optional_value(options, "probe_step_index")?, + }) + } +} + +fn optional_value FromPyObject<'py>>( + options: Option<&Bound<'_, PyDict>>, + name: &str, +) -> PyResult> { + let value: Option> = match options { + Some(options) => options.get_item(name)?, + None => None, + }; + match value { + Some(value) if !value.is_none() => Ok(Some(value.extract()?)), + _ => Ok(None), + } +} + +/// Comparison SQL of each planned SQL test, or of one chain step's actual-output probe. +#[pyfunction] +#[pyo3(signature = (test_entries, **options))] +fn render_sql_test_comparison_sql( + py: Python<'_>, + test_entries: Vec>, + options: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + rendered_comparisons(py, &test_entries, &RenderSettings::read(options)?) +} + +fn rendered_comparisons( + py: Python<'_>, + test_entries: &[Bound<'_, PyAny>], + settings: &RenderSettings, +) -> PyResult> { + if test_entries.is_empty() { + return Ok(Vec::new()); + } + let mut requests: Vec = Vec::with_capacity(test_entries.len()); + for entry in test_entries { + let mut request: Value = comparison_request( + entry, + &settings.set_difference_operator, + settings.sql_analysis_dialect.as_deref(), + )?; + if let (Some(index), Some(fields)) = (settings.probe_step_index, request.as_object_mut()) { + fields.insert("probeStepIndex".to_owned(), json!(index)); + } + requests.push(request); + } + let request_json: String = + serde_json::to_string(&json!({"requests": requests, "workers": RENDER_WORKERS})) + .map_err(value_error)?; + let response: String = py + .compiler_detach(|| render_json(&request_json)) + .map_err(value_error)?; + let rendered: Vec = serde_json::from_str(&response).map_err(value_error)?; + rendered + .iter() + .map(|item| { + item.get("sql") + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| value_error("native SQL-test rendering returned an invalid result")) + }) + .collect() +} + +/// The comparison SQL of one planned SQL test under `adapter`'s operator and dialect. +pub(crate) fn single_comparison_sql( + entry: &Bound<'_, PyAny>, + adapter: &Bound<'_, PyAny>, +) -> PyResult { + let settings = RenderSettings { + set_difference_operator: adapter + .call_method0("render_set_difference_operator")? + .extract()?, + sql_analysis_dialect: adapter.call_method0("sql_analysis_dialect")?.extract()?, + probe_step_index: None, + }; + rendered_comparisons(entry.py(), std::slice::from_ref(entry), &settings)? + .into_iter() + .next() + .ok_or_else(|| value_error("native SQL-test rendering returned an invalid batch response")) +} + +/// The native comparison render request of one planned SQL test. +pub(crate) fn comparison_request( + entry: &Bound<'_, PyAny>, + set_difference_operator: &str, + sql_analysis_dialect: Option<&str>, +) -> PyResult { + let chain: Vec = entry + .getattr("chain")? + .try_iter()? + .map(|step| chain_step_request(&step?)) + .collect::>()?; + let assertions: Vec = entry + .getattr("assertions")? + .try_iter()? + .map(|assertion| { + let assertion: Bound<'_, PyAny> = assertion?; + Ok(json!({ + "name": assertion.getattr("name")?.extract::()?, + "resolvedSql": assertion.getattr("resolved_sql")?.extract::()?, + "liftedCtes": assertion.getattr("lifted_ctes")?.extract::>()?, + "comparisonBodySql": assertion.getattr("comparison_body_sql")?.extract::>()?, + })) + }) + .collect::>()?; + Ok(json!({ + "chain": chain, + "assertions": assertions, + "sqlAnalysisEnabled": entry.getattr("sql_analysis_enabled")?.extract::()?, + "setDifferenceOperator": set_difference_operator, + "sqlAnalysisDialect": sql_analysis_dialect, + })) +} + +/// The native request of one planned chain step. +pub(crate) fn chain_step_request(step: &Bound<'_, PyAny>) -> PyResult { + Ok(json!({ + "modelName": step.getattr("model_name")?.extract::()?, + "resolvedSql": step.getattr("resolved_sql")?.extract::()?, + "expectedCteSql": step.getattr("expected_cte_sql")?.extract::>()?, + "liftedCtes": step.getattr("lifted_ctes")?.extract::>()?, + "comparisonBodySql": step.getattr("comparison_body_sql")?.extract::>()?, + "expectedColumns": step.getattr("expected_columns")?.extract::>>()?, + "expectedLiftedCtes": step.getattr("expected_lifted_ctes")?.extract::>()?, + })) +} + +/// A bounded query for one direction of an expected-output difference of one chain step. +#[pyfunction] +#[pyo3(signature = (**options))] +fn render_sql_test_difference_sample_sql( + py: Python<'_>, + options: Option<&Bound<'_, PyDict>>, +) -> PyResult { + let request = Request::new("render_sql_test_difference_sample_sql", options)?; + let test_entry: Bound<'_, PyAny> = request.get("test_entry")?; + let step: Bound<'_, PyAny> = request.get("step")?; + if step.getattr("expected_cte_sql")?.is_none() { + return Ok(String::new()); + } + let settings: RenderSettings = RenderSettings::read(options)?; + let request_json: String = serde_json::to_string(&json!({ + "step": chain_step_request(&step)?, + "sqlAnalysisEnabled": test_entry.getattr("sql_analysis_enabled")?.extract::()?, + "setDifferenceOperator": settings.set_difference_operator, + "sqlAnalysisDialect": settings.sql_analysis_dialect, + "direction": request.get("direction")?.extract::()?, + "sampleLimit": request.get("sample_limit")?.extract::()?, + "useTopClause": settings.sql_analysis_dialect.as_deref() == Some(TOP_CLAUSE_DIALECT), + })) + .map_err(value_error)?; + let response: String = py + .compiler_detach(|| render_difference_sample_json(&request_json)) + .map_err(value_error)?; + let rendered: Value = serde_json::from_str(&response).map_err(value_error)?; + rendered + .get("sql") + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| value_error("native SQL-test difference rendering returned invalid SQL")) +} + +/// Describe listed expected columns absent from casefolded model output names. +#[pyfunction] +#[pyo3(signature = (*, model_name, expected_columns, available_columns))] +pub(crate) fn describe_missing_expected_columns( + model_name: &str, + expected_columns: Vec, + available_columns: Bound<'_, PyAny>, +) -> PyResult> { + let available: Vec = available_columns + .try_iter()? + .map(|column| column?.extract::()) + .collect::>()?; + Ok(missing_expected_columns( + model_name, + &expected_columns, + &available, + )) +} + +/// SQL comparing a scenario's actual output with its expected SQL. +#[pyfunction] +#[pyo3(signature = (*, actual_sql, expected_sql, set_difference_operator))] +pub(crate) fn build_scenario_expected_comparison_sql( + actual_sql: &str, + expected_sql: &str, + set_difference_operator: &str, +) -> String { + scenario_expected_comparison_sql(actual_sql, expected_sql, set_difference_operator) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_sql_test_comparison_sql, module)?)?; + module.add_function(wrap_pyfunction!( + render_sql_test_difference_sample_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(describe_missing_expected_columns, module)?)?; + module.add_function(wrap_pyfunction!( + build_scenario_expected_comparison_sql, + module + )?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs index 46f16013cc..63025f70be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs @@ -6,24 +6,6 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::types::CompilerDetach; -#[pyfunction] -fn render_sql_test_comparisons_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::compiler::main::sql_test_rendering::render_json(request_json) - }) - .map_err(value_error) -} - -#[pyfunction] -fn render_sql_test_difference_sample_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::compiler::main::sql_test_difference_sampling::render_difference_sample_json( - request_json, - ) - }) - .map_err(value_error) -} - #[pyfunction] fn extract_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult { py.compiler_detach(|| { @@ -33,11 +15,6 @@ fn extract_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult) -> PyResult<()> { - module.add_function(wrap_pyfunction!(render_sql_test_comparisons_json, module)?)?; - module.add_function(wrap_pyfunction!( - render_sql_test_difference_sample_json, - module - )?)?; module.add_function(wrap_pyfunction!(extract_sql_tests_json, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 3e6257e344..cd267b1829 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -13,6 +13,7 @@ use sqlbuild_cache::artifacts::main::new_sql_test_record::new_sql_test_record; use sqlbuild_cache::artifacts::main::reusable_sql_test_record::reusable_sql_test_record; use sqlbuild_cache::artifacts::models::SqlTestArtifactRecord; +use crate::bindings::_helpers::analysis::sql_test_comparisons::single_comparison_sql; use crate::bindings::_helpers::compile_command::artifact_batch::{ArtifactBatch, timed}; use crate::bindings::_helpers::compile_command::reuse_recording::{ ArtifactRecorder, CompileArtifactWrites, artifact_kept, artifact_moved, artifact_tree_moved, @@ -50,7 +51,6 @@ const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; const MODEL_CHAIN_MODULE: &str = "sqlbuild._native"; const PLAN_ARTIFACTS_MODULE: &str = "sqlbuild._native"; -const COMPARISON_SQL_MODULE: &str = "sqlbuild.executor.testing.main.comparison_sql"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const TARGET_OPTIONS: (&[&str], &[&str]) = (&["target_dir", "adapter", "project"], &["manifest"]); const STAGING_CHANGED_HELP: &str = @@ -271,21 +271,7 @@ fn write_planned_target<'py>( .div(TESTS_DIR)? .div(planned_sql_test_path(&entry)?)?; let comparison_sql: String = timed(py, "comparison_render_ms", || { - call_with( - &imported(py, COMPARISON_SQL_MODULE, "build_sql_test_comparison_sql")?, - &[ - ("test_entry", entry.clone()), - ( - "set_difference_operator", - adapter.call_method0("render_set_difference_operator")?, - ), - ( - "sql_analysis_dialect", - adapter.call_method0("sql_analysis_dialect")?, - ), - ], - )? - .extract() + single_comparison_sql(&entry, adapter) })?; batch.queue_sql(&path, &comparison_sql)?; managed.push(path); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs index 8e8784396f..8a6a95628b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs @@ -14,10 +14,10 @@ use sqlbuild_analysis::target_paths::main::sql_test_output_path::sql_test_output use sqlbuild_analysis::target_paths::models::SqlTestPathFacts; use sqlbuild_core::text::main::python_isspace::python_isspace; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::analysis::sql_test_comparisons::single_comparison_sql; +use crate::bindings::_helpers::compile_driver::python_calls::imported; const LIFECYCLE_TYPES: &str = "sqlbuild.adapter.contract.types"; -const COMPARISON_SQL_MODULE: &str = "sqlbuild.executor.testing.main.comparison_sql"; const RUN_DIR: &str = "run"; const TESTS_DIR: &str = "tests"; const CHECKS_DIR: &str = "checks"; @@ -172,7 +172,6 @@ pub(crate) fn write_test_runtime_target( plan: &Bound<'_, PyAny>, results: &Bound<'_, PyAny>, ) -> PyResult<()> { - let py: Python<'_> = target_dir.py(); let mut executed: HashSet = HashSet::new(); for result in results.try_iter()? { executed.insert(test_key(&result?, "test_name")?); @@ -183,21 +182,7 @@ pub(crate) fn write_test_runtime_target( if !executed.contains(&test_key(&entry, "name")?) { continue; } - let sql: String = call_with( - &imported(py, COMPARISON_SQL_MODULE, "build_sql_test_comparison_sql")?, - &[ - ("test_entry", entry.clone()), - ( - "set_difference_operator", - adapter.call_method0("render_set_difference_operator")?, - ), - ( - "sql_analysis_dialect", - adapter.call_method0("sql_analysis_dialect")?, - ), - ], - )? - .extract()?; + let sql: String = single_comparison_sql(&entry, adapter)?; write_text( &tests_dir.div(planned_sql_test_path(&entry)?)?, &format!("{}\n", sql.trim_end_matches(python_isspace)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 88011119fd..7fd07624a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -35,10 +35,10 @@ use crate::bindings::_helpers::planner_identity::metadata::{ compiled_function_fingerprint_sql, function_local_hashes, model_declared_columns_hash, model_version_identity_metadata_json, }; +use crate::bindings::_helpers::type_system::equality::types_equal as native_types_equal; const NATIVE_MODULE: &str = "sqlbuild._native"; const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; -const TYPES_EQUAL_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; const DYNAMIC_FAMILIES_MODULE: &str = "sqlbuild._native"; const DEBUG_EVENT_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; const LOG_SQL_MODULE: &str = "sqlbuild.diagnostics.main.log_sql"; @@ -594,18 +594,8 @@ fn schema_findings_of<'py>( ) -> PyResult> { let py: Python<'py> = dialect.py(); let dialect_name: Option = dialect.extract()?; - let types_equal_function: Bound<'py, PyAny> = imported(py, TYPES_EQUAL_MODULE, "types_equal")?; - let mut types_equal = |left: &str, right: &str| -> PyResult { - call_with( - &types_equal_function, - &[ - ("left", PyString::new(py, left).into_any()), - ("right", PyString::new(py, right).into_any()), - ("dialect", dialect.clone()), - ], - )? - .is_truthy() - }; + let mut types_equal = + |left: &str, right: &str| -> PyResult { native_types_equal(dialect, left, right) }; let matching_function: Bound<'py, PyAny> = imported(py, DYNAMIC_FAMILIES_MODULE, "matching_dynamic_families")?; let mut single_family_type = |name: &str| -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs index 748fa0f48c..4898e1db42 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs @@ -1,11 +1,10 @@ //! Source contract validation and adapter-controlled projection rendering. -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, -}; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_resolution::values::{cursor_wrap, method, string}; use crate::bindings::_helpers::planner_strategy::values::{PLANNER_EXCEPTIONS_MODULE, input_error}; use crate::bindings::_helpers::references::source_relations::render_source_relation; +use crate::bindings::_helpers::type_system::equality::types_equal; use pyo3::IntoPyObject; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -16,8 +15,6 @@ use sqlbuild_sqltext::resolution::main::{ reference_spans::reference_spans, source_alias::source_alias, }; -const TYPES_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; - fn source_error(py: Python<'_>, message: &str, code: &str) -> PyErr { input_error( py, @@ -86,15 +83,11 @@ fn check_columns( if actual.is_empty() { continue; } - let equal = call_with( - &imported(py, TYPES_MODULE, "types_equal")?, - &[ - ("left", string(py, actual)), - ("right", string(py, &kind)), - ("dialect", adapter.getattr("sql_analysis_dialect_name")?), - ], - )? - .is_truthy()?; + let equal: bool = types_equal( + &adapter.getattr("sql_analysis_dialect_name")?, + actual, + &kind, + )?; if !equal { return Err(source_error( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs index 1ac9d93180..595e088c2d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs @@ -7,12 +7,12 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::text::main::python_casefold::python_casefold; use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::type_system::equality::types_equal as native_types_equal; const ADOPTION_MODELS_MODULE: &str = "sqlbuild.compiler.contract_adoption.models"; const ADOPTION_TYPES_MODULE: &str = "sqlbuild.compiler.contract_adoption.types"; const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const TYPES_EQUAL_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; const DYNAMIC_FAMILIES_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; const MODEL_RESOURCE: &str = "model"; @@ -241,16 +241,11 @@ fn types_equal( adapter: &Bound<'_, PyAny>, (left, right): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), ) -> PyResult { - let py: Python<'_> = adapter.py(); - keyword_call( - &py.import(TYPES_EQUAL_MODULE)?.getattr("types_equal")?, - vec![ - ("left", left.clone()), - ("right", right.clone()), - ("dialect", adapter.call_method0("sql_analysis_dialect")?), - ], - )? - .is_truthy() + native_types_equal( + &adapter.call_method0("sql_analysis_dialect")?, + &left.extract::()?, + &right.extract::()?, + ) } /// The finding for a physical column no declared column names, if any. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs index 1e2e9b28c9..410a105496 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs @@ -7,4 +7,5 @@ pub(crate) mod integration_records; pub(crate) mod integration_values; pub(crate) mod projection; pub(crate) mod registration; +pub(crate) mod resource_ids; pub(crate) mod terminal_index; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs index e7f3325b56..1cc2df689e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs @@ -3,8 +3,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; use pyo3::types::{PyFloat, PyInt}; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::plan_output::values::text; +use crate::bindings::_helpers::result_output::resource_ids::{audit_identity, sql_test_identity}; use crate::bindings::_helpers::result_output::terminal_index::{TerminalEventIndex, current_index}; const RUN_MODELS: &str = "sqlbuild.executor.run.models"; @@ -14,8 +15,6 @@ const PYTHON_NODE_MODELS: &str = "sqlbuild.executor.python_nodes.models"; const TESTING_MODELS: &str = "sqlbuild.executor.testing.models"; const AUDITING_MODELS: &str = "sqlbuild.executor.auditing.models"; const CLONE_MODELS: &str = "sqlbuild.executor.clone.models"; -const SQL_TEST_ID_MODULE: &str = "sqlbuild.executor.testing.main._resource_id"; -const AUDIT_ID_MODULE: &str = "sqlbuild.executor.auditing.main.resource_id"; /// The executor result types that result output distinguishes. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -72,35 +71,12 @@ pub(crate) fn result_kind(result: &Bound<'_, PyAny>) -> PyResult { /// `sql_test_resource_id` of one SQL test result. pub(crate) fn sql_test_check_id(result: &Bound<'_, PyAny>) -> PyResult { - let fields: Vec<(&str, Bound<'_, PyAny>)> = - ["test_name", "source_path", "block_index", "case_name"] - .into_iter() - .map(|name| Ok((name, result.getattr(name)?))) - .collect::>()?; - call_with( - &imported(result.py(), SQL_TEST_ID_MODULE, "sql_test_resource_id")?, - &fields, - )? - .extract() + sql_test_identity(result, "test_name") } /// `audit_resource_id` of one audit result. pub(crate) fn audit_check_id(result: &Bound<'_, PyAny>) -> PyResult { - let fields: Vec<(&str, Bound<'_, PyAny>)> = [ - ("audit_name", "audit_name"), - ("attachment_kind", "attachment_kind"), - ("attached_target_kind", "attached_target_kind"), - ("attached_target_name", "attached_target_name"), - ("attached_column_name", "attached_column_name"), - ] - .into_iter() - .map(|(keyword, name)| Ok((keyword, result.getattr(name)?))) - .collect::>()?; - call_with( - &imported(result.py(), AUDIT_ID_MODULE, "audit_resource_id")?, - &fields, - )? - .extract() + audit_identity(result) } /// The canonical resource name and identifier of an executor result. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs index 890cf21372..5392910355 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs @@ -3,11 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::result_output::{ - documents, event_writer, integration_envelope, terminal_index, + documents, event_writer, integration_envelope, resource_ids, terminal_index, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { terminal_index::register(module)?; + resource_ids::register(module)?; integration_envelope::register(module)?; event_writer::register(module)?; documents::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs new file mode 100644 index 0000000000..7e3c8f4812 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs @@ -0,0 +1,104 @@ +//! Canonical execution-protocol identities of SQL test cases and attached audits. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use sqlbuild_core::nested_progress::main::audit_resource_id::audit_resource_id as audit_id; +use sqlbuild_core::nested_progress::main::sql_test_resource_id::sql_test_resource_id as sql_test_id; + +/// The exact stable identity of one plain or parameterised SQL test. +#[pyfunction] +#[pyo3(signature = (*, test_name, source_path, block_index, case_name))] +pub(crate) fn sql_test_resource_id( + test_name: &str, + source_path: Bound<'_, PyAny>, + block_index: Option, + case_name: Option, +) -> PyResult { + let path: Option = if source_path.is_none() { + None + } else { + Some(source_path.call_method0("as_posix")?.extract()?) + }; + Ok(match (path, block_index, case_name) { + (Some(path), Some(block), Some(case)) => { + sql_test_id(test_name, Some((&path, block, &case))) + } + _ => sql_test_id(test_name, None), + }) +} + +/// The stable identity of one attached audit execution. +#[pyfunction] +#[pyo3(signature = (**options))] +fn audit_resource_id(options: Option<&Bound<'_, PyDict>>) -> PyResult { + let request = Request::new("audit_resource_id", options)?; + let target_kind: Option> = request + .optional("attached_target_kind")? + .filter(|kind| !kind.is_none()); + let kind: Bound<'_, PyAny> = match target_kind { + Some(kind) => kind, + None => request.get("attachment_kind")?, + }; + audit_identity_of( + &request.get("audit_name")?, + &kind, + ( + &request.get("attached_target_name")?, + &request.get("attached_column_name")?, + ), + ) +} + +fn audit_identity_of( + audit_name: &Bound<'_, PyAny>, + kind: &Bound<'_, PyAny>, + (target, column): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult { + let target: Option = target.extract()?; + let column: Option = column.extract()?; + Ok(audit_id( + &audit_name.extract::()?, + &kind.getattr("value")?.extract::()?, + (target.as_deref(), column.as_deref()), + )) +} + +/// `sql_test_resource_id` of one result or plan entry carrying `test_name_field` and case fields. +pub(crate) fn sql_test_identity( + result: &Bound<'_, PyAny>, + test_name_field: &str, +) -> PyResult { + sql_test_resource_id( + &result.getattr(test_name_field)?.extract::()?, + result.getattr("source_path")?, + result.getattr("block_index")?.extract()?, + result.getattr("case_name")?.extract()?, + ) +} + +/// `audit_resource_id` of one audit result. +pub(crate) fn audit_identity(result: &Bound<'_, PyAny>) -> PyResult { + let target_kind: Bound<'_, PyAny> = result.getattr("attached_target_kind")?; + let kind: Bound<'_, PyAny> = if target_kind.is_none() { + result.getattr("attachment_kind")? + } else { + target_kind + }; + audit_identity_of( + &result.getattr("audit_name")?, + &kind, + ( + &result.getattr("attached_target_name")?, + &result.getattr("attached_column_name")?, + ), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(sql_test_resource_id, module)?)?; + module.add_function(wrap_pyfunction!(audit_resource_id, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 8163395c66..30d5cc9aef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -8,7 +8,7 @@ use pyo3::exceptions::{PyOSError, PyTypeError, PyValueError}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; +use pyo3::types::{PyDict, PyStringMethods, PyTuple}; use pyo3::{ FromPyObject, IntoPyObject, IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction, }; @@ -34,15 +34,12 @@ use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::rules::findings::{FindingClasses, finding_fault}; use crate::bindings::_helpers::rules::selection::rules_error; use crate::bindings::_helpers::rules::sql_test_objects::{scenario_fact, sql_test_fact}; +use crate::bindings::_helpers::type_system::equality::types_equal; use crate::bindings::types::CompilerDetach; -const LEFT_ARGUMENT: &str = "left"; -const RIGHT_ARGUMENT: &str = "right"; -const DIALECT_ARGUMENT: &str = "dialect"; const NATIVE_MODULE: &str = "sqlbuild._native"; const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const CODE_IDENTITY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiled_code_identity"; -const TYPES_EQUAL_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; const DATACLASSES_MODULE: &str = "dataclasses"; const NATIVE_RULES_CACHE_FILE: &str = "native.json"; const NATIVE_RULES_MEMO_FILE: &str = "native-response.json"; @@ -67,7 +64,6 @@ struct ProjectRows<'py> { public_enums: Vec>, public_constants: Vec>, initial_findings: Vec>, - types_equal: Bound<'py, PyAny>, dialect: Bound<'py, PyAny>, } @@ -166,7 +162,6 @@ fn built_request(options: &Request<'_, '_>) -> PyResult { .get("initial_findings")? .try_iter()? .collect::>()?, - types_equal: imported(py, TYPES_EQUAL_MODULE, "types_equal")?, dialect, }; let facts: Bound<'_, NativeCompiledProject> = @@ -349,19 +344,6 @@ fn as_rules_error(py: Python<'_>, result: PyResult) -> PyResult { }) } -/// `left=`, `right=` and `dialect=`: `types_equal` takes keyword-only arguments. -fn type_arguments<'py>( - dialect: &Bound<'py, PyAny>, - left: &str, - right: &str, -) -> PyResult> { - let arguments: Bound<'py, PyDict> = PyDict::new(dialect.py()); - arguments.set_item(LEFT_ARGUMENT, left)?; - arguments.set_item(RIGHT_ARGUMENT, right)?; - arguments.set_item(DIALECT_ARGUMENT, dialect)?; - Ok(arguments) -} - fn built_models( project: &ProjectRows<'_>, request: &EvaluateRequest, @@ -369,10 +351,7 @@ fn built_models( ) -> PyResult> { let mut python_error: Option = None; let fallback = |left: &str, right: &str| -> Result { - match type_arguments(&project.dialect, left, right) - .and_then(|arguments| project.types_equal.call((), Some(&arguments))) - .and_then(|equal| equal.extract::()) - { + match types_equal(&project.dialect, left, right) { Ok(equal) => Ok(equal), Err(error) => { let message: String = error.to_string(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs index 43e3e99094..486212aaf4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs @@ -1,5 +1,6 @@ //! Scenario executor results read into plain rows and runtime artifact inputs. +use sqlbuild_analysis::compiler::main::scenario_expected_comparison_sql::scenario_expected_comparison_sql; use std::collections::HashSet; use std::path::{Path, PathBuf}; @@ -15,7 +16,6 @@ use sqlbuild_core::scenario_command::models::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::scenario_command::setup::text; const SUCCESS_STATUS: &str = "success"; const FAILED_STATUS: &str = "failed"; @@ -28,7 +28,6 @@ const LOCAL_DIR: &str = "local"; const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; const QUALIFIED_NAME_MODULE: &str = "sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name"; -const COMPARISON_SQL_MODULE: &str = "sqlbuild.executor.scenario.main.expected_comparison_sql"; fn items<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { value.try_iter()?.collect() @@ -262,25 +261,13 @@ fn expectation_sql( ], )? .extract()?; - let sql: String = call_with( - &imported( - py, - COMPARISON_SQL_MODULE, - "build_scenario_expected_comparison_sql", - )?, - &[ - ( - "actual_sql", - text(py, &format!("SELECT * FROM {actual_relation}")), - ), - ("expected_sql", expected.getattr("expected_sql")?), - ( - "set_difference_operator", - adapter.call_method0("render_set_difference_operator")?, - ), - ], - )? - .extract()?; + let sql: String = scenario_expected_comparison_sql( + &format!("SELECT * FROM {actual_relation}"), + &expected.getattr("expected_sql")?.extract::()?, + &adapter + .call_method0("render_set_difference_operator")? + .extract::()?, + ); expectations .expected .push((expected.getattr("model_name")?.extract()?, sql)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index df0faa4614..3a6644efe0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -30,6 +30,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::compile_outputs::runtime_target::write_test_runtime_target; use crate::bindings::_helpers::result_output::integration_values::json_value; +use crate::bindings::_helpers::result_output::resource_ids::sql_test_identity; const NATIVE_MODULE: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; @@ -39,7 +40,6 @@ const TEST_PLAN_MODULE: &str = "sqlbuild._native"; const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; const EXECUTOR_PIPELINE_MODELS: &str = "sqlbuild.executor.pipeline.models"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; -const RESOURCE_ID_MODULE: &str = "sqlbuild.executor.testing.main._resource_id"; const DECIMAL_KIND: &str = "decimal"; const PASS_OUTCOME: &str = "pass"; const EXPECT_LABEL: &str = "expect"; @@ -218,15 +218,7 @@ impl TestCallbackRelay { if group.is_empty() { group = UNKNOWN_GROUP.to_owned(); } - let resource_id: Bound<'_, PyAny> = call_with( - &imported(py, RESOURCE_ID_MODULE, "sql_test_resource_id")?, - &[ - ("test_name", result.getattr("test_name")?), - ("source_path", result.getattr("source_path")?), - ("block_index", result.getattr("block_index")?), - ("case_name", result.getattr("case_name")?), - ], - )?; + let resource_id: Bound<'_, PyAny> = text(py, &sql_test_identity(result, "test_name")?); call_with( &self.progress.bind(py).getattr("on_item_complete")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs index 4f39f527d9..8aa749df36 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs @@ -3,7 +3,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple}; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::analysis::sql_test_comparisons::describe_missing_expected_columns; +use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::sql_test_glue::constants::{COMPILE_TYPES, PLAN_MODELS}; use crate::bindings::_helpers::sql_test_glue::constants::{TABLE_FN, UDF}; use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; @@ -147,21 +148,14 @@ pub(super) fn expected_diagnostics( if expected.is_none() || available.is_none() { continue; } - let message = call_with( - &imported( - py, - "sqlbuild.executor.testing.main._missing_expected_columns", - "describe_missing_expected_columns", - )?, - &[ - ("model_name", name.clone()), - ("expected_columns", expected), - ("available_columns", available), - ], - )?; - if message.is_none() { + let Some(message) = describe_missing_expected_columns( + &name.extract::()?, + expected.extract()?, + available, + )? + else { continue; - } + }; let location = super::fixture_validation::fixture_location( test, &resource_types.getattr("SQL_TEST")?, @@ -171,7 +165,7 @@ pub(super) fn expected_diagnostics( "SQL test '{}': {}: {}", test.getattr("name")?.extract::()?, location, - message.extract::()? + message )); } Ok(result) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/equality.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/equality.rs new file mode 100644 index 0000000000..3af3c77ea6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/equality.rs @@ -0,0 +1,90 @@ +//! Semantic type equality for native callers, with the adapter wrapper's parse-fallback debug event. + +use std::collections::HashMap; +use std::sync::{Mutex, OnceLock, PoisonError}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_analysis::type_system::main::normalize_type::normalize_type; +use sqlbuild_analysis::type_system::models::{NormalizedType, TypeNormalization}; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::type_system::normalization::normalization_error; + +const DEBUG_EVENT_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; +const TYPE_NORMALIZATION_LOGGER: &str = "sqlbuild.adapter"; +const TYPE_PARSE_FALLBACK_MESSAGE: &str = "type normalization polyglot parse failed; falling back"; +const DEFAULT_DIALECT: &str = "generic"; +const CACHE_LIMIT: usize = 1024; + +type Cache = Mutex>; + +static NORMALIZED: OnceLock = OnceLock::new(); + +/// Whether two type strings are semantically equal under `dialect` (a `TypeDialect`, name or `None`). +pub(crate) fn types_equal(dialect: &Bound<'_, PyAny>, left: &str, right: &str) -> PyResult { + let name: String = if dialect.is_truthy()? { + dialect.str()?.extract()? + } else { + DEFAULT_DIALECT.to_owned() + }; + let left_type: NormalizedType = normalized(dialect, &name, left)?; + let right_type: NormalizedType = normalized(dialect, &name, right)?; + Ok(left_type == right_type) +} + +fn normalized(dialect: &Bound<'_, PyAny>, name: &str, type_sql: &str) -> PyResult { + let py: Python<'_> = dialect.py(); + let cache: &Cache = NORMALIZED.get_or_init(|| Mutex::new(HashMap::new())); + let key: (String, String) = (type_sql.to_owned(), name.to_owned()); + let cached: Option = cache + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(&key) + .cloned(); + if let Some(found) = cached { + return Ok(found); + } + let TypeNormalization { + normalized, + parse_error, + } = compiler_guard(|| Ok(normalize_type(type_sql, name)))? + .map_err(|error| normalization_error(py, &error))?; + if let Some(parse_error) = parse_error { + log_parse_fallback(dialect, type_sql, &parse_error)?; + } + let mut held = cache.lock().unwrap_or_else(PoisonError::into_inner); + if held.len() >= CACHE_LIMIT { + held.clear(); + } + held.insert(key, normalized.clone()); + Ok(normalized) +} + +fn log_parse_fallback( + dialect: &Bound<'_, PyAny>, + type_sql: &str, + parse_error: &str, +) -> PyResult<()> { + let py: Python<'_> = dialect.py(); + call_with( + &imported(py, DEBUG_EVENT_MODULE, "log_debug_event")?, + &[ + ( + "logger", + py.import("logging")? + .getattr("getLogger")? + .call1((TYPE_NORMALIZATION_LOGGER,))?, + ), + ( + "message", + PyString::new(py, TYPE_PARSE_FALLBACK_MESSAGE).into_any(), + ), + ("type_sql", PyString::new(py, type_sql).into_any()), + ("dialect", dialect.str()?.into_any()), + ("sqlbuild_error", PyString::new(py, parse_error).into_any()), + ], + )?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs index bcc09d3157..30a05c9b4f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod equality; pub(crate) mod normalization; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 4c229c0e09..4ff394572a 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -28,6 +28,7 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { analysis::normalization::register(module)?; analysis::queries::register(module)?; analysis::sql_tests::register(module)?; + analysis::sql_test_comparisons::register(module)?; sqltext::registration::register(module)?; boundary::oracles::register(module)?; boundary::expected_errors::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 93df92569d..3ef8fb438d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -174,6 +174,9 @@ from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( AuditPlanEntry, ChainStep, + BackfillResult, + ChainStep, + ChangeDetectionResult, CloneSourcePlanEntry, ColumnMigrationPlanEntry, CursorBounds, @@ -4910,6 +4913,39 @@ def scope_test_ctes( ) -> list[tuple[str | None, list[tuple[str, str]]]]: ... def omitted_ceremonial_select(sql: str, syntax: dict[str, object]) -> int | None: ... def extract_sql_tests_json(request_json: str) -> str: ... +def render_sql_test_comparison_sql( + test_entries: Sequence[SqlTestPlanEntry], + *, + set_difference_operator: str = "EXCEPT", + sql_analysis_dialect: str | None = None, + probe_step_index: int | None = None, +) -> list[str]: ... +def render_sql_test_difference_sample_sql( + *, + test_entry: SqlTestPlanEntry, + step: ChainStep, + direction: str, + sample_limit: int, + set_difference_operator: str = "EXCEPT", + sql_analysis_dialect: str | None = None, +) -> str: ... +def describe_missing_expected_columns( + *, model_name: str, expected_columns: Sequence[str], available_columns: Iterable[str] +) -> str | None: ... +def build_scenario_expected_comparison_sql( + *, actual_sql: str, expected_sql: str, set_difference_operator: str +) -> str: ... +def sql_test_resource_id( + *, test_name: str, source_path: PurePath | None, block_index: int | None, case_name: str | None +) -> str: ... +def audit_resource_id( + *, + audit_name: str, + attachment_kind: AuditAttachmentKind, + attached_target_kind: AttachedAuditTargetKind | None = None, + attached_target_name: str | None, + attached_column_name: str | None, +) -> str: ... def validated_model_cursor_intrinsics( *, sql: str, config_values: Mapping[str, object], model_name: str ) -> str: ... diff --git a/src/sqlbuild/executor/auditing/main/resource_id.py b/src/sqlbuild/executor/auditing/main/resource_id.py deleted file mode 100644 index 52de73a259..0000000000 --- a/src/sqlbuild/executor/auditing/main/resource_id.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Canonical resource identity for an attached audit execution.""" - -from sqlbuild.compiler.auditing.types import AuditAttachmentKind -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind - - -def audit_resource_id( - *, - audit_name: str, - attachment_kind: AuditAttachmentKind, - attached_target_kind: AttachedAuditTargetKind | None = None, - attached_target_name: str | None, - attached_column_name: str | None, -) -> str: - """Format the stable execution-protocol identity of one attached audit.""" - - identity_kind: str = ( - attached_target_kind.value if attached_target_kind is not None else attachment_kind.value - ) - - parts: tuple[str | None, ...] = ( - "audit", - audit_name, - identity_kind, - attached_target_name, - attached_column_name, - ) - return ":".join(filter(None, parts)) diff --git a/src/sqlbuild/executor/build/_helpers/end_audits.py b/src/sqlbuild/executor/build/_helpers/end_audits.py index f19dcfac46..3169b537d9 100644 --- a/src/sqlbuild/executor/build/_helpers/end_audits.py +++ b/src/sqlbuild/executor/build/_helpers/end_audits.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import audit_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope from sqlbuild.compiler.compile.models import CompiledRelationLocation @@ -14,7 +15,6 @@ from sqlbuild.executor.auditing.main.publish_completed_audit_results import ( publish_completed_audit_results, ) -from sqlbuild.executor.auditing.main.resource_id import audit_resource_id from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.runtime.observability.classes.resource_attempt_lifecycle import ( ResourceAttemptLifecycle, diff --git a/src/sqlbuild/executor/build/_helpers/source_audits.py b/src/sqlbuild/executor/build/_helpers/source_audits.py index b1c6b2fcc7..17da4ee61c 100644 --- a/src/sqlbuild/executor/build/_helpers/source_audits.py +++ b/src/sqlbuild/executor/build/_helpers/source_audits.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import audit_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope from sqlbuild.compiler.compile.models import CompiledObjectKey @@ -15,7 +16,6 @@ from sqlbuild.executor.auditing.main.publish_completed_audit_results import ( publish_completed_audit_results, ) -from sqlbuild.executor.auditing.main.resource_id import audit_resource_id from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build._helpers.blocking import downstream_blocked_keys from sqlbuild.executor.build.models import SourceAuditRunResult diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 892158d337..6284493d5c 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -15,7 +15,7 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, sql_test_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import RelationInfo @@ -108,7 +108,6 @@ from sqlbuild.executor.seed.main._execute import execute_seed from sqlbuild.executor.testing.constants import SQL_TEST_ENTRY_MISSING_CODE from sqlbuild.executor.testing.main._execute import execute_sql_test -from sqlbuild.executor.testing.main._resource_id import sql_test_resource_id from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome from sqlbuild.microbatches.classes.direct_store import ( diff --git a/src/sqlbuild/executor/pipeline/_helpers/auditing.py b/src/sqlbuild/executor/pipeline/_helpers/auditing.py index aee09c8380..9cc0dbe756 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/auditing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/auditing.py @@ -15,7 +15,7 @@ from typing import Any, cast from uuid import uuid4 -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, audit_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput @@ -29,7 +29,6 @@ from sqlbuild.executor.auditing.main.publish_completed_audit_results import ( publish_completed_audit_results, ) -from sqlbuild.executor.auditing.main.resource_id import audit_resource_id from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.pipeline._helpers.connections import ( close_connections, diff --git a/src/sqlbuild/executor/pipeline/_helpers/testing.py b/src/sqlbuild/executor/pipeline/_helpers/testing.py index bf5ebd668b..81577014c0 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/testing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/testing.py @@ -12,6 +12,7 @@ from typing import Any, cast from uuid import uuid4 +from sqlbuild._native import sql_test_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import FunctionInfo @@ -31,7 +32,6 @@ from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.constants import SQL_TEST_EXECUTION_ERROR_CODE from sqlbuild.executor.testing.main._execute import execute_sql_test -from sqlbuild.executor.testing.main._resource_id import sql_test_resource_id from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.executor.testing.types import SqlTestOutcome from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/executor/scenario/main/_expected_expectations.py b/src/sqlbuild/executor/scenario/main/_expected_expectations.py index 196ab590c0..d848eb1477 100644 --- a/src/sqlbuild/executor/scenario/main/_expected_expectations.py +++ b/src/sqlbuild/executor/scenario/main/_expected_expectations.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import build_scenario_expected_comparison_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, @@ -14,9 +15,6 @@ SCENARIO_EXEC_EXPECTED_FAILED, SCENARIO_EXEC_EXPECTED_INTERNAL, ) -from sqlbuild.executor.scenario.main.expected_comparison_sql import ( - build_scenario_expected_comparison_sql, -) from sqlbuild.executor.scenario.models import ScenarioExpectedExpectationExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/src/sqlbuild/executor/scenario/main/expected_comparison_sql.py b/src/sqlbuild/executor/scenario/main/expected_comparison_sql.py deleted file mode 100644 index b8ceff07f7..0000000000 --- a/src/sqlbuild/executor/scenario/main/expected_comparison_sql.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Public scenario expected-output comparison SQL entrypoint.""" - -from __future__ import annotations - - -def build_scenario_expected_comparison_sql( - *, - actual_sql: str, - expected_sql: str, - set_difference_operator: str, -) -> str: - """Build SQL that compares actual scenario output to expected SQL.""" - - return ( - "WITH " - f"__actual AS ({actual_sql}), " - f"__expected AS ({expected_sql}) " - "SELECT " - "(SELECT COUNT(*) FROM __actual) AS actual_count, " - "(SELECT COUNT(*) FROM __expected) AS expected_count, " - "(SELECT COUNT(*) FROM (" - f"SELECT * FROM __actual {set_difference_operator} SELECT * FROM __expected" - ") AS __sqlbuild_mismatch) AS mismatched_count" - ) diff --git a/src/sqlbuild/executor/testing/_helpers/difference_samples.py b/src/sqlbuild/executor/testing/_helpers/difference_samples.py index f7b3fe9141..70bb3a2911 100644 --- a/src/sqlbuild/executor/testing/_helpers/difference_samples.py +++ b/src/sqlbuild/executor/testing/_helpers/difference_samples.py @@ -4,28 +4,19 @@ import logging from dataclasses import replace -from typing import Any, Final, cast +from typing import Any, Final -import orjson - -import sqlbuild._native as _native +from sqlbuild._native import render_sql_test_difference_sample_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor from sqlbuild.diagnostics.main.log_debug_event import log_debug_event -from sqlbuild.executor.testing._helpers.native_requests import chain_step_request -from sqlbuild.executor.testing.exceptions import SqlTestRenderingError from sqlbuild.executor.testing.models import ( SqlTestColumnDifference, SqlTestDifferenceSample, StepResult, ) -from sqlbuild.executor.testing.types import ( - NativeSqlTestRenderingModule, - SqlTestDifferenceDirection, - SqlTestOutcome, -) +from sqlbuild.executor.testing.types import SqlTestDifferenceDirection, SqlTestOutcome _ROW_LIMIT: Final[int] = 3 _COLUMN_LIMIT: Final[int] = 12 @@ -110,39 +101,6 @@ def _align_single_difference( ) -def build_sql_test_difference_sample_sql( - *, - test_entry: SqlTestPlanEntry, - step: ChainStep, - direction: SqlTestDifferenceDirection, - sample_limit: int, - set_difference_operator: str = "EXCEPT", - sql_analysis_dialect: str | None = None, -) -> str: - """Build a bounded query for one direction of an expected-output difference.""" - - if step.expected_cte_sql is None: - return "" - request: dict[str, object] = { - "step": chain_step_request(step=step), - "sqlAnalysisEnabled": test_entry.sql_analysis_enabled, - "setDifferenceOperator": set_difference_operator, - "sqlAnalysisDialect": sql_analysis_dialect, - "direction": direction.value, - "sampleLimit": sample_limit, - "useTopClause": sql_analysis_dialect == TypeDialect.TSQL, - } - response: object = orjson.loads( - cast(NativeSqlTestRenderingModule, _native).render_sql_test_difference_sample_json( - orjson.dumps(request, option=orjson.OPT_SORT_KEYS).decode() - ) - ) - sql: object = response.get("sql") if isinstance(response, dict) else None - if not isinstance(sql, str): - raise SqlTestRenderingError("native SQL-test difference rendering returned invalid SQL") - return sql - - def _fetch_difference_samples( *, test_entry: SqlTestPlanEntry, @@ -151,7 +109,7 @@ def _fetch_difference_samples( adapter: BaseAdapter, connection: Any, ) -> tuple[SqlTestDifferenceSample, ...]: - sql: str = build_sql_test_difference_sample_sql( + sql: str = render_sql_test_difference_sample_sql( test_entry=test_entry, step=step, direction=direction, diff --git a/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py b/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py index 9c193493fe..c82b524ce9 100644 --- a/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py +++ b/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py @@ -3,20 +3,13 @@ from __future__ import annotations import logging -from typing import Any, cast +from typing import Any -import orjson - -import sqlbuild._native as _native +from sqlbuild._native import describe_missing_expected_columns, render_sql_test_comparison_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import SqlTestPlanEntry from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor from sqlbuild.diagnostics.main.log_debug_event import log_debug_event -from sqlbuild.executor.testing._helpers.native_requests import comparison_request -from sqlbuild.executor.testing.main._missing_expected_columns import ( - describe_missing_expected_columns, -) -from sqlbuild.executor.testing.types import NativeSqlTestRenderingModule _LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") @@ -47,20 +40,13 @@ def missing_expected_columns_message( def _probe_actual_columns( *, test_entry: SqlTestPlanEntry, step_index: int, adapter: BaseAdapter, connection: Any ) -> frozenset[str] | None: - request: dict[str, object] = comparison_request( - test_entry=test_entry, + (sql,) = render_sql_test_comparison_sql( + (test_entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), + probe_step_index=step_index, ) - request["probeStepIndex"] = step_index - response: object = orjson.loads( - cast(NativeSqlTestRenderingModule, _native).render_sql_test_comparisons_json( - orjson.dumps({"requests": [request]}, option=orjson.OPT_SORT_KEYS).decode() - ) - ) - item: object = response[0] if isinstance(response, list) and response else None - sql: object = item.get("sql") if isinstance(item, dict) else None - if not isinstance(sql, str) or not sql: + if not sql: return None try: cursor: Any = adapter.execute( diff --git a/src/sqlbuild/executor/testing/_helpers/native_requests.py b/src/sqlbuild/executor/testing/_helpers/native_requests.py deleted file mode 100644 index 37e3566c0d..0000000000 --- a/src/sqlbuild/executor/testing/_helpers/native_requests.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Serialize planned SQL-test steps for the native comparison renderer.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.models import ChainStep, SqlTestAssertionStep, SqlTestPlanEntry - - -def comparison_request( - *, - test_entry: SqlTestPlanEntry, - set_difference_operator: str, - sql_analysis_dialect: str | None, -) -> dict[str, object]: - """Serialize one planned test into a native comparison render request.""" - - return { - "chain": [chain_step_request(step=step) for step in test_entry.chain], - "assertions": [ - assertion_step_request(assertion=assertion) for assertion in test_entry.assertions - ], - "sqlAnalysisEnabled": test_entry.sql_analysis_enabled, - "setDifferenceOperator": set_difference_operator, - "sqlAnalysisDialect": sql_analysis_dialect, - } - - -def chain_step_request(*, step: ChainStep) -> dict[str, object]: - """Serialize one planned chain step for the native renderer.""" - - return { - "modelName": step.model_name, - "resolvedSql": step.resolved_sql, - "expectedCteSql": step.expected_cte_sql, - "liftedCtes": step.lifted_ctes, - "comparisonBodySql": step.comparison_body_sql, - "expectedColumns": step.expected_columns, - "expectedLiftedCtes": step.expected_lifted_ctes, - } - - -def assertion_step_request(*, assertion: SqlTestAssertionStep) -> dict[str, object]: - """Serialize one planned assertion step for the native renderer.""" - - return { - "name": assertion.name, - "resolvedSql": assertion.resolved_sql, - "liftedCtes": assertion.lifted_ctes, - "comparisonBodySql": assertion.comparison_body_sql, - } diff --git a/src/sqlbuild/executor/testing/exceptions.py b/src/sqlbuild/executor/testing/exceptions.py deleted file mode 100644 index 6b18460b4c..0000000000 --- a/src/sqlbuild/executor/testing/exceptions.py +++ /dev/null @@ -1,5 +0,0 @@ -"""SQL test execution exceptions.""" - - -class SqlTestRenderingError(RuntimeError): - """Raised when native SQL-test rendering returns an invalid response.""" diff --git a/src/sqlbuild/executor/testing/main/_execute.py b/src/sqlbuild/executor/testing/main/_execute.py index 59a5d59020..f0c31495c7 100644 --- a/src/sqlbuild/executor/testing/main/_execute.py +++ b/src/sqlbuild/executor/testing/main/_execute.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import render_sql_test_comparison_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.planner.models import SqlTestPlanEntry @@ -20,7 +21,6 @@ from sqlbuild.executor.testing.main._sql_length import ( validate_unit_test_sql_length, ) -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.executor.testing.types import SqlTestOutcome from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -36,8 +36,8 @@ def execute_sql_test( ) -> SqlTestExecutionResult: """Execute one SQL unit test and collect bounded diagnostics for failed comparisons.""" - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=test_entry, + (comparison_sql,) = render_sql_test_comparison_sql( + (test_entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), ) diff --git a/src/sqlbuild/executor/testing/main/_missing_expected_columns.py b/src/sqlbuild/executor/testing/main/_missing_expected_columns.py deleted file mode 100644 index b8536007a2..0000000000 --- a/src/sqlbuild/executor/testing/main/_missing_expected_columns.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Explain expected-output columns that a model does not produce.""" - -from __future__ import annotations - -from sqlbuild.executor.testing.constants import SQL_TEST_IDENTIFIER_QUOTES - - -def describe_missing_expected_columns( - *, - model_name: str, - expected_columns: tuple[str, ...], - available_columns: frozenset[str], -) -> str | None: - """Describe listed expected columns absent from casefolded model output names.""" - - missing: tuple[str, ...] = tuple( - column - for column in expected_columns - if _normalized_identifier(column) not in available_columns - ) - if not missing: - return None - return ( - f"expected output '{model_name}' lists columns that model '{model_name}' does not " - f"output: {', '.join(missing)}. Expected CTEs are compared only on the columns they " - "list, matched by name" - ) - - -def _normalized_identifier(identifier: str) -> str: - closing_quote: str | None = SQL_TEST_IDENTIFIER_QUOTES.get(identifier[:1]) - if closing_quote is None or not identifier.endswith(closing_quote): - return identifier.casefold() - return identifier[1:-1].replace(closing_quote * 2, closing_quote).casefold() diff --git a/src/sqlbuild/executor/testing/main/_resource_id.py b/src/sqlbuild/executor/testing/main/_resource_id.py deleted file mode 100644 index 6d15602496..0000000000 --- a/src/sqlbuild/executor/testing/main/_resource_id.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Canonical resource identity for one SQL test case execution.""" - -from pathlib import Path - - -def sql_test_resource_id( - *, - test_name: str, - source_path: Path | None, - block_index: int | None, - case_name: str | None, -) -> str: - """Return the exact stable identity of one plain or parameterized SQL test.""" - - if source_path is None or block_index is None or case_name is None: - return f"sql_test:{test_name}" - return f"sql_test:{source_path.as_posix()}:{block_index}:{case_name}" diff --git a/src/sqlbuild/executor/testing/main/comparison_sql.py b/src/sqlbuild/executor/testing/main/comparison_sql.py deleted file mode 100644 index 099f3c7613..0000000000 --- a/src/sqlbuild/executor/testing/main/comparison_sql.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Build executable SQL unit-test comparison queries.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.models import SqlTestPlanEntry -from sqlbuild.executor.testing.main.comparison_sql_batch import ( - build_sql_test_comparison_sql_batch, -) - - -def build_sql_test_comparison_sql( - *, - test_entry: SqlTestPlanEntry, - set_difference_operator: str = "EXCEPT", - sql_analysis_dialect: str | None = None, -) -> str: - """Build the single SQL statement used to execute a SQL unit test.""" - - (comparison_sql,) = build_sql_test_comparison_sql_batch( - test_entries=(test_entry,), - set_difference_operator=set_difference_operator, - sql_analysis_dialect=sql_analysis_dialect, - ) - return comparison_sql diff --git a/src/sqlbuild/executor/testing/main/comparison_sql_batch.py b/src/sqlbuild/executor/testing/main/comparison_sql_batch.py deleted file mode 100644 index 1ba1d0d9b0..0000000000 --- a/src/sqlbuild/executor/testing/main/comparison_sql_batch.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Build batches of executable SQL unit-test comparison queries natively.""" - -from __future__ import annotations - -from typing import Any, cast - -import orjson - -import sqlbuild._native as _native -from sqlbuild.compiler.planner.models import SqlTestPlanEntry -from sqlbuild.executor.testing._helpers.native_requests import comparison_request -from sqlbuild.executor.testing.constants import SQL_TEST_NATIVE_RENDER_WORKERS -from sqlbuild.executor.testing.exceptions import SqlTestRenderingError -from sqlbuild.executor.testing.types import NativeSqlTestRenderingModule - - -def build_sql_test_comparison_sql_batch( - *, - test_entries: tuple[SqlTestPlanEntry, ...], - set_difference_operator: str = "EXCEPT", - sql_analysis_dialect: str | None = None, -) -> tuple[str, ...]: - """Build comparison SQL for a deterministic batch of planned SQL tests.""" - - if not test_entries: - return () - requests: list[dict[str, object]] = [ - comparison_request( - test_entry=test_entry, - set_difference_operator=set_difference_operator, - sql_analysis_dialect=sql_analysis_dialect, - ) - for test_entry in test_entries - ] - response: object = orjson.loads( - cast(NativeSqlTestRenderingModule, _native).render_sql_test_comparisons_json( - orjson.dumps( - {"requests": requests, "workers": SQL_TEST_NATIVE_RENDER_WORKERS}, - option=orjson.OPT_SORT_KEYS, - ).decode() - ) - ) - if not isinstance(response, list) or len(response) != len(test_entries): - raise SqlTestRenderingError("native SQL-test rendering returned an invalid batch response") - rendered: list[str] = [] - for item in response: - item_dict: dict[str, Any] | None = ( - cast(dict[str, Any], item) if isinstance(item, dict) else None - ) - sql: object = item_dict.get("sql") if item_dict is not None else None - if not isinstance(sql, str): - raise SqlTestRenderingError("native SQL-test rendering returned an invalid result") - rendered.append(sql) - return tuple(rendered) diff --git a/src/sqlbuild/executor/testing/types.py b/src/sqlbuild/executor/testing/types.py index f26fb0dea4..b0dc160aaa 100644 --- a/src/sqlbuild/executor/testing/types.py +++ b/src/sqlbuild/executor/testing/types.py @@ -3,15 +3,6 @@ from __future__ import annotations from enum import StrEnum -from typing import Protocol - - -class NativeSqlTestRenderingModule(Protocol): - """Native SQL-test comparison-rendering boundary.""" - - def render_sql_test_comparisons_json(self, request_json: str) -> str: ... - - def render_sql_test_difference_sample_json(self, request_json: str) -> str: ... class SqlTestOutcome(StrEnum): diff --git a/tests/integration/src/sqlbuild/executor/testing/helpers.py b/tests/integration/src/sqlbuild/executor/testing/helpers.py index 77801bf3be..2514914c86 100644 --- a/tests/integration/src/sqlbuild/executor/testing/helpers.py +++ b/tests/integration/src/sqlbuild/executor/testing/helpers.py @@ -6,13 +6,13 @@ from pathlib import Path from typing import Any +from sqlbuild._native import render_sql_test_comparison_sql from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from sqlbuild.executor.testing.main._execute import execute_sql_test -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome from tests.integration.src.sqlbuild.compiler.pipeline.helpers import ( @@ -148,11 +148,11 @@ def _project_test_step( ] step: ChainStep = {item.model_name: item for item in entry.chain}[model_name] build_sql: str = {model.name: model.query_sql for model in result.project.models}[model_name] - rendered_sql: str = build_sql_test_comparison_sql( - test_entry=entry, + rendered_sql: str = render_sql_test_comparison_sql( + (entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] return build_sql, step.comparison_body_sql or step.resolved_sql, rendered_sql diff --git a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_build_parity.py b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_build_parity.py index f00fdeaf43..10a88288a7 100644 --- a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_build_parity.py +++ b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_build_parity.py @@ -9,9 +9,9 @@ import pytest +from sqlbuild._native import render_sql_test_comparison_sql from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.planner.models import SqlTestPlanEntry -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.integration.src.sqlbuild.executor.testing._test_types import ( ExpectedBooleanTestCase, SqlTestBuildParityTestCase, @@ -124,8 +124,8 @@ def test_given_shared_cte_name_in_chain_when_rendering_tsql_then_rename_runs_lik ) -> None: entry: SqlTestPlanEntry = build_shared_cte_name_chain_entry() - renamed_sql: str = build_sql_test_comparison_sql(test_entry=entry, sql_analysis_dialect="tsql") - nested_sql: str = build_sql_test_comparison_sql(test_entry=entry, sql_analysis_dialect="duckdb") + renamed_sql: str = render_sql_test_comparison_sql((entry,), sql_analysis_dialect="tsql")[0] + nested_sql: str = render_sql_test_comparison_sql((entry,), sql_analysis_dialect="duckdb")[0] assert "__sqb_cte_0 AS (SELECT id + 1 AS id FROM __ref__stg)" in renamed_sql assert "__ref__mart AS (SELECT __sqb_cte_0.id FROM __sqb_cte_0)" in renamed_sql diff --git a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py index 1bad0a901d..5af2fe7cee 100644 --- a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py +++ b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py @@ -7,18 +7,12 @@ import pytest +from sqlbuild._native import render_sql_test_comparison_sql, render_sql_test_difference_sample_sql from sqlbuild.adapter.contract.types import StatementSizeLimit from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry -from sqlbuild.executor.testing._helpers.difference_samples import ( - build_sql_test_difference_sample_sql, -) from sqlbuild.executor.testing.main._execute import execute_sql_test -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql -from sqlbuild.executor.testing.main.comparison_sql_batch import ( - build_sql_test_comparison_sql_batch, -) from sqlbuild.executor.testing.models import ( SqlTestColumnDifference, SqlTestExecutionResult, @@ -113,12 +107,14 @@ def execute(self, *, connection: Any, sql: str) -> Any: def test_given_sql_test_chain_when_building_comparison_sql_then_uses_readable_ctes( test_case: SqlTestComparisonSqlTestCase, ) -> None: - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=build_sql_test_plan_entry( - name="test_model", - chain_steps=test_case.chain_steps, + comparison_sql: str = render_sql_test_comparison_sql( + ( + build_sql_test_plan_entry( + name="test_model", + chain_steps=test_case.chain_steps, + ), ) - ) + )[0] expected_fragment: str for expected_fragment in test_case.expected_fragments: @@ -196,10 +192,12 @@ def test_given_conflicting_preanalyzed_ctes_when_executing_then_each_step_uses_i adapter=adapter, connection=connection, ) - (native_sql,) = build_sql_test_comparison_sql_batch( - test_entries=(entry,), - set_difference_operator=adapter.render_set_difference_operator(), - sql_analysis_dialect=adapter.sql_analysis_dialect(), + (native_sql,) = tuple( + render_sql_test_comparison_sql( + (entry,), + set_difference_operator=adapter.render_set_difference_operator(), + sql_analysis_dialect=adapter.sql_analysis_dialect(), + ) ) native_rows: list[tuple[object, ...]] = connection.execute(native_sql).fetchall() @@ -229,7 +227,7 @@ def test_given_sqlserver_dialect_when_building_sample_sql_then_uses_bounded_top_ dialect: str | None = SqlServerAdapter().sql_analysis_dialect() assert dialect == test_case.dialect - sample_sql: str = build_sql_test_difference_sample_sql( + sample_sql: str = render_sql_test_difference_sample_sql( test_entry=entry, step=entry.chain[0], direction=SqlTestDifferenceDirection.UNEXPECTED, diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py index fb7ea24f5d..82059d5a6d 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -109,6 +109,7 @@ def test_given_native_source_when_checking_module_paths_then_reports_errors( description="native crates import only existing Python modules and attributes", expected_errors=(), expected_logger_names=( + "sqlbuild.adapter", "sqlbuild.cli", "sqlbuild.cli.commands._helpers.cost.collection", "sqlbuild.cli.commands.classes.prepared_compile_artifacts", diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 4b59180665..80ddcd32f0 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -15,6 +15,7 @@ build_manifest, compiled_model_record, project_with, + render_sql_test_comparison_sql, write_compile_target, write_static_compile_target, ) @@ -27,7 +28,6 @@ DiagnosticPhase, ) from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( ExpectedMessageTestCase, TargetWriterCacheTestCase, @@ -74,7 +74,7 @@ def test_given_plan_output_when_writing_target_then_expected_files_are_written( ) -> None: plan_output: PlanOutput = build_target_writer_plan_output() expected_test_sql: str = ( - build_sql_test_comparison_sql(test_entry=plan_output.test_entries[0]) + "\n" + render_sql_test_comparison_sql((plan_output.test_entries[0],))[0] + "\n" ) manifest: _native.NativeManifest = build_manifest( project=build_test_project(), diff --git a/tests/unit/src/sqlbuild/cli/output/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/output/_helpers/helpers.py index a6be058340..c8143028a1 100644 --- a/tests/unit/src/sqlbuild/cli/output/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/output/_helpers/helpers.py @@ -10,12 +10,12 @@ ExecutionEventWriter, IntegrationResultEnvelope, TerminalEventIndex, + audit_resource_id, format_audit_execution_json, format_build_execution_json, format_test_execution_json, terminal_event_index_scope, ) -from sqlbuild.executor.auditing.main.resource_id import audit_resource_id from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 395b2862df..60fc1cb492 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -16,6 +16,7 @@ plan_compiled_sql_test_artifacts, plan_sql_tests, plan_sql_tests_natively, + render_sql_test_comparison_sql, sql_test_plan_error_messages, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -50,7 +51,6 @@ from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( PlanTestChainTestCase, ) @@ -142,11 +142,11 @@ def assert_native_artifact_matches_runtime_plan( """Assert the compiled artifact matches the runtime comparison SQL for the same plan.""" adapter: DuckDbAdapter = DuckDbAdapter() - expected_sql: str = build_sql_test_comparison_sql( - test_entry=entry, + expected_sql: str = render_sql_test_comparison_sql( + (entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] (artifact,) = plan_compiled_sql_test_artifacts( project=project, tests=(sql_test,), diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index ae3cce2643..94d3b06183 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -13,6 +13,7 @@ build_test_command_plan, plan_compiled_sql_test_artifacts, project_with, + render_sql_test_comparison_sql, resolve_static_command_scope, resolve_static_relation_context, sql_test_plan_error_messages, @@ -41,6 +42,7 @@ PlanOutput, SqlTestPlanEntry, ) +from sqlbuild.compiler.planner.types import WarningSeverity from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( AssertionChainCteErrorTestCase, @@ -1472,11 +1474,11 @@ def test_given_adapter_function_when_resolving_with_analysis_then_keeps_authored entry, _ = plan_single_test( test=compiled_test, project=project, adapter=adapter, sql_analysis_enabled=True ) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=entry, + comparison_sql: str = render_sql_test_comparison_sql( + (entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] assert test_case.expected_sql_fragment in entry.chain[0].resolved_sql assert test_case.expected_sql_fragment in comparison_sql @@ -1523,11 +1525,11 @@ def test_given_tsql_shared_cte_name_when_planning_then_helper_and_model_cte_stay entry, _ = plan_single_test( test=compiled_test, project=project, adapter=adapter, sql_analysis_enabled=True ) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=entry, + comparison_sql: str = render_sql_test_comparison_sql( + (entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] for expected_fragment in test_case.expected_sql_fragments.values(): assert expected_fragment in comparison_sql, comparison_sql diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py index 50b8999d87..30097490c8 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py @@ -21,6 +21,7 @@ from sqlbuild._native import ( AuditProgressReporter, CostContext, + audit_resource_id, StatementLedgerEntry, read_statement_ledger, record_statement, @@ -36,7 +37,7 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput -from sqlbuild.cost.models import CostResourceContext +from sqlbuild.cost.models import CostResourceContext, StatementLedgerEntry from sqlbuild.executor.auditing.main.resource_id import audit_resource_id from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.pipeline._helpers import auditing diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/test_expected_comparison_sql.py b/tests/unit/src/sqlbuild/executor/scenario/main/test_expected_comparison_sql.py index 2509e08f22..62200c7067 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/test_expected_comparison_sql.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/test_expected_comparison_sql.py @@ -2,9 +2,7 @@ import pytest -from sqlbuild.executor.scenario.main.expected_comparison_sql import ( - build_scenario_expected_comparison_sql, -) +from sqlbuild._native import build_scenario_expected_comparison_sql from tests.unit.src.sqlbuild.executor.scenario.main._test_types import ( ScenarioExpectedComparisonSqlTestCase, ) diff --git a/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py b/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py index 33923872f2..6f7a19a455 100644 --- a/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py +++ b/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py @@ -4,12 +4,9 @@ import pytest +from sqlbuild._native import render_sql_test_comparison_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql -from sqlbuild.executor.testing.main.comparison_sql_batch import ( - build_sql_test_comparison_sql_batch, -) from tests.unit.src.sqlbuild.executor.testing.main._test_types import ( BuildComparisonSqlTestCase, ExpectedBooleanTestCase, @@ -72,11 +69,11 @@ def test_given_adapter_when_building_comparison_sql_then_it_uses_expected_set_di ) -> None: adapter: BaseAdapter = build_comparison_test_adapter(test_case.adapter_name) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=build_comparison_test_entry(), + comparison_sql: str = render_sql_test_comparison_sql( + (build_comparison_test_entry(),), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] expected_fragment: str for expected_fragment in test_case.expected_fragments: @@ -119,11 +116,11 @@ def test_given_snowflake_function_when_lifting_comparison_ctes_then_keeps_author ), ) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=entry, + comparison_sql: str = render_sql_test_comparison_sql( + (entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] for expected_fragment in test_case.expected_fragments: assert expected_fragment in comparison_sql @@ -161,11 +158,11 @@ def test_given_sql_analysis_disabled_when_building_comparison_sql_then_step_sql_ ), ) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=entry, + comparison_sql: str = render_sql_test_comparison_sql( + (entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] expected_fragment: str for expected_fragment in test_case.expected_fragments: @@ -193,11 +190,11 @@ def test_given_assertion_step_when_building_comparison_sql_then_it_counts_failin ) -> None: adapter: BaseAdapter = build_comparison_test_adapter(test_case.adapter_name) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=build_assertion_test_entry(), + comparison_sql: str = render_sql_test_comparison_sql( + (build_assertion_test_entry(),), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] expected_fragment: str for expected_fragment in test_case.expected_fragments: @@ -224,11 +221,11 @@ def test_given_matching_helper_ctes_when_building_comparison_sql_then_lifts_them ) -> None: adapter: BaseAdapter = build_comparison_test_adapter(test_case.adapter_name) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=build_comparison_test_entry_with_helper_ctes(), + comparison_sql: str = render_sql_test_comparison_sql( + (build_comparison_test_entry_with_helper_ctes(),), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] expected_fragment: str for expected_fragment in test_case.expected_fragments: @@ -250,9 +247,9 @@ def test_given_matching_helper_ctes_when_building_comparison_sql_then_lifts_them def test_given_unasserted_transitive_steps_when_building_comparison_then_emits_only_expected_model( test_case: BuildComparisonSqlTestCase, ) -> None: - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=build_transitive_comparison_test_entry(), - ) + comparison_sql: str = render_sql_test_comparison_sql( + (build_transitive_comparison_test_entry(),) + )[0] for expected_fragment in test_case.expected_fragments: assert expected_fragment in comparison_sql @@ -276,11 +273,11 @@ def test_given_databricks_table_fn_when_building_comparison_sql_then_preserves_c ) -> None: adapter: BaseAdapter = build_comparison_test_adapter(test_case.adapter_name) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=build_table_function_test_entry(), + comparison_sql: str = render_sql_test_comparison_sql( + (build_table_function_test_entry(),), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] expected_fragment: str for expected_fragment in test_case.expected_fragments: @@ -305,13 +302,17 @@ def test_given_bigquery_table_fn_when_building_comparison_sql_then_preserves_bac ) -> None: adapter: BaseAdapter = build_comparison_test_adapter(test_case.adapter_name) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=build_table_function_test_entry( - resolved_sql=("SELECT * FROM `project-d5f92072-d107-4987-9ef.test.customer_orders`(1)") + comparison_sql: str = render_sql_test_comparison_sql( + ( + build_table_function_test_entry( + resolved_sql=( + "SELECT * FROM `project-d5f92072-d107-4987-9ef.test.customer_orders`(1)" + ) + ), ), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] expected_fragment: str for expected_fragment in test_case.expected_fragments: @@ -346,10 +347,7 @@ def test_given_preanalyzed_step_with_authored_cte_when_building_comparison_then_ ), ) - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=entry, - sql_analysis_dialect="tsql", - ) + comparison_sql: str = render_sql_test_comparison_sql((entry,), sql_analysis_dialect="tsql")[0] assert ( comparison_sql.index("__ref__raw_orders AS") < comparison_sql.index("picked AS") @@ -373,13 +371,12 @@ def test_given_representative_plans_when_rendering_batch_then_matches_single_ren build_assertion_test_entry(), ) expected: tuple[str, ...] = tuple( - build_sql_test_comparison_sql(test_entry=entry, sql_analysis_dialect="duckdb") + render_sql_test_comparison_sql((entry,), sql_analysis_dialect="duckdb")[0] for entry in entries ) - actual: tuple[str, ...] = build_sql_test_comparison_sql_batch( - test_entries=entries, - sql_analysis_dialect="duckdb", + actual: tuple[str, ...] = tuple( + render_sql_test_comparison_sql(entries, sql_analysis_dialect="duckdb") ) assert (actual == expected) is test_case.expected_result @@ -397,16 +394,18 @@ def test_given_quoted_table_function_when_rendering_batch_then_matches_single_re resolved_sql="SELECT * FROM `project-d5f92072-d107-4987-9ef.test.customer_orders`(1)" ) adapter: BaseAdapter = build_comparison_test_adapter("bigquery") - expected: str = build_sql_test_comparison_sql( - test_entry=entry, + expected: str = render_sql_test_comparison_sql( + (entry,), set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) + )[0] - (actual,) = build_sql_test_comparison_sql_batch( - test_entries=(entry,), - set_difference_operator=adapter.render_set_difference_operator(), - sql_analysis_dialect=adapter.sql_analysis_dialect(), + (actual,) = tuple( + render_sql_test_comparison_sql( + (entry,), + set_difference_operator=adapter.render_set_difference_operator(), + sql_analysis_dialect=adapter.sql_analysis_dialect(), + ) ) assert (actual == expected) is test_case.expected_result From f923970e560db9457233b5e5609003beba5ea89d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:19:47 +0100 Subject: [PATCH 02/11] refactor: fit relation names and merge microbatch intervals natively Planner scenario naming no longer calls back into Python to fit artifact logical names; the executor imports the native fitting functions. The executor's raw microbatch interval merge uses the native cursor algebra in place of the duplicate Python implementation. --- .../src/identifier_fitting/_helpers/bytes.rs | 20 +++++ .../src/identifier_fitting/_helpers/mod.rs | 1 + .../src/identifier_fitting/constants.rs | 4 + .../main/fit_artifact_logical_name.rs | 41 +++++++++ .../main/fit_auxiliary_relation_name.rs | 31 +++++++ .../src/identifier_fitting/main/mod.rs | 2 + .../src/identifier_fitting/mod.rs | 5 ++ crates/sqlbuild-analysis/src/lib.rs | 1 + .../src/bindings/_helpers/cursor/intervals.rs | 84 ++++++++++++++++++- .../src/bindings/_helpers/cursor/scalars.rs | 2 +- .../src/bindings/_helpers/mod.rs | 1 + .../planner_registration/registration.rs | 5 +- .../_helpers/planner_scenario/artifacts.rs | 44 +++++----- .../_helpers/relation_naming/fitting.rs | 63 ++++++++++++++ .../bindings/_helpers/relation_naming/mod.rs | 2 + .../_helpers/relation_naming/registration.rs | 9 ++ src/sqlbuild/_native.pyi | 9 ++ .../relations/_helpers/identifier_fitting.py | 63 -------------- src/sqlbuild/adapter/relations/constants.py | 1 - .../main/fit_artifact_logical_name.py | 18 ---- .../main/fit_auxiliary_relation_name.py | 15 ---- .../janitor/_helpers/archive_names.py | 2 +- .../executor/load/_helpers/staging.py | 2 +- .../run/_helpers/execution/table_targets.py | 2 +- .../_helpers/materializations/incremental.py | 7 +- .../_helpers/materializations/microbatch.py | 10 +-- .../run/_helpers/materializations/snapshot.py | 2 +- .../_helpers/validation/type_enforcement.py | 2 +- .../microbatches/main/merge_intervals.py | 73 ---------------- .../test_fit_artifact_logical_name.py | 2 +- .../test_fit_auxiliary_relation_name.py | 4 +- 31 files changed, 311 insertions(+), 216 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/identifier_fitting/_helpers/bytes.rs create mode 100644 crates/sqlbuild-analysis/src/identifier_fitting/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/identifier_fitting/constants.rs create mode 100644 crates/sqlbuild-analysis/src/identifier_fitting/main/fit_artifact_logical_name.rs create mode 100644 crates/sqlbuild-analysis/src/identifier_fitting/main/fit_auxiliary_relation_name.rs create mode 100644 crates/sqlbuild-analysis/src/identifier_fitting/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/identifier_fitting/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/relation_naming/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/relation_naming/registration.rs delete mode 100644 src/sqlbuild/adapter/relations/_helpers/identifier_fitting.py delete mode 100644 src/sqlbuild/adapter/relations/main/fit_artifact_logical_name.py delete mode 100644 src/sqlbuild/adapter/relations/main/fit_auxiliary_relation_name.py delete mode 100644 src/sqlbuild/microbatches/main/merge_intervals.py diff --git a/crates/sqlbuild-analysis/src/identifier_fitting/_helpers/bytes.rs b/crates/sqlbuild-analysis/src/identifier_fitting/_helpers/bytes.rs new file mode 100644 index 0000000000..03373d0092 --- /dev/null +++ b/crates/sqlbuild-analysis/src/identifier_fitting/_helpers/bytes.rs @@ -0,0 +1,20 @@ +//! UTF-8 byte truncation and short digests. + +use sha2::{Digest, Sha256}; + +use crate::identifier_fitting::constants::SHORTENED_NAME_HASH_LENGTH; + +/// The longest prefix whose UTF-8 encoding fits in `max_bytes`, never splitting a character. +pub(crate) fn truncate_bytes(value: &str, max_bytes: usize) -> &str { + let mut end: usize = max_bytes.min(value.len()); + while !value.is_char_boundary(end) { + end -= 1; + } + &value[..end] +} + +/// The first hex digits of the SHA-256 digest of `value`. +pub(crate) fn short_hash(value: &str) -> String { + let digest: String = format!("{:x}", Sha256::digest(value.as_bytes())); + digest[..SHORTENED_NAME_HASH_LENGTH].to_owned() +} diff --git a/crates/sqlbuild-analysis/src/identifier_fitting/_helpers/mod.rs b/crates/sqlbuild-analysis/src/identifier_fitting/_helpers/mod.rs new file mode 100644 index 0000000000..14e49498ab --- /dev/null +++ b/crates/sqlbuild-analysis/src/identifier_fitting/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod bytes; diff --git a/crates/sqlbuild-analysis/src/identifier_fitting/constants.rs b/crates/sqlbuild-analysis/src/identifier_fitting/constants.rs new file mode 100644 index 0000000000..8d0306b3f6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/identifier_fitting/constants.rs @@ -0,0 +1,4 @@ +//! Identifier fitting constants. + +/// Hex digits of the SHA-256 digest a shortened name keeps. +pub(crate) const SHORTENED_NAME_HASH_LENGTH: usize = 8; diff --git a/crates/sqlbuild-analysis/src/identifier_fitting/main/fit_artifact_logical_name.rs b/crates/sqlbuild-analysis/src/identifier_fitting/main/fit_artifact_logical_name.rs new file mode 100644 index 0000000000..9aa50e4a15 --- /dev/null +++ b/crates/sqlbuild-analysis/src/identifier_fitting/main/fit_artifact_logical_name.rs @@ -0,0 +1,41 @@ +//! Fit a readable logical name after a fixed prefix. + +use crate::identifier_fitting::_helpers::bytes::{short_hash, truncate_bytes}; +use crate::identifier_fitting::constants::SHORTENED_NAME_HASH_LENGTH; + +/// The logical name, or its truncation with a hash suffix; `Err` carries the input error message. +pub fn fit_artifact_logical_name( + (logical_name, fixed_prefix): (&str, &str), + identifier_limit: i64, + artifact_label: &str, +) -> Result { + let max_logical_length: i64 = identifier_limit - byte_length(fixed_prefix); + if max_logical_length < 1 { + return Err(format!( + "{artifact_label} prefix '{fixed_prefix}' does not fit within identifier limit {identifier_limit}" + )); + } + if byte_length(logical_name) <= max_logical_length { + return Ok(logical_name.to_owned()); + } + let suffix_length: i64 = byte_length_of(SHORTENED_NAME_HASH_LENGTH) + 1; + if max_logical_length <= suffix_length { + return Err(format!( + "{artifact_label} name for '{logical_name}' cannot fit within identifier limit {identifier_limit}" + )); + } + let kept: usize = usize::try_from(max_logical_length - suffix_length).unwrap_or(0); + Ok(format!( + "{}_{}", + truncate_bytes(logical_name, kept), + short_hash(logical_name) + )) +} + +fn byte_length(value: &str) -> i64 { + byte_length_of(value.len()) +} + +fn byte_length_of(length: usize) -> i64 { + i64::try_from(length).unwrap_or(i64::MAX) +} diff --git a/crates/sqlbuild-analysis/src/identifier_fitting/main/fit_auxiliary_relation_name.rs b/crates/sqlbuild-analysis/src/identifier_fitting/main/fit_auxiliary_relation_name.rs new file mode 100644 index 0000000000..36389b76d4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/identifier_fitting/main/fit_auxiliary_relation_name.rs @@ -0,0 +1,31 @@ +//! Append a suffix to a relation name within an identifier limit. + +use crate::identifier_fitting::_helpers::bytes::{short_hash, truncate_bytes}; + +/// `base_name + suffix`, shortening the base with a stable hash when too long; `Err` carries the message. +pub fn fit_auxiliary_relation_name( + base_name: &str, + suffix: &str, + identifier_limit: i64, +) -> Result { + let whole: i64 = length(base_name) + length(suffix); + if whole <= identifier_limit { + return Ok(format!("{base_name}{suffix}")); + } + let hash_part: String = format!("_{}", short_hash(base_name)); + let kept_length: i64 = identifier_limit - length(suffix) - length(&hash_part); + if kept_length < 1 { + return Err(format!( + "auxiliary relation suffix '{suffix}' for '{base_name}' cannot fit within identifier limit {identifier_limit}" + )); + } + let kept: usize = usize::try_from(kept_length).unwrap_or(0); + Ok(format!( + "{}{hash_part}{suffix}", + truncate_bytes(base_name, kept) + )) +} + +fn length(value: &str) -> i64 { + i64::try_from(value.len()).unwrap_or(i64::MAX) +} diff --git a/crates/sqlbuild-analysis/src/identifier_fitting/main/mod.rs b/crates/sqlbuild-analysis/src/identifier_fitting/main/mod.rs new file mode 100644 index 0000000000..21551a11b4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/identifier_fitting/main/mod.rs @@ -0,0 +1,2 @@ +pub mod fit_artifact_logical_name; +pub mod fit_auxiliary_relation_name; diff --git a/crates/sqlbuild-analysis/src/identifier_fitting/mod.rs b/crates/sqlbuild-analysis/src/identifier_fitting/mod.rs new file mode 100644 index 0000000000..526e5bf21f --- /dev/null +++ b/crates/sqlbuild-analysis/src/identifier_fitting/mod.rs @@ -0,0 +1,5 @@ +//! Fit generated relation names within identifier limits measured in UTF-8 bytes. + +pub(crate) mod _helpers; +mod constants; +pub mod main; diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 1b29ef8804..7e636a4508 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -15,6 +15,7 @@ pub mod diff_selection; pub mod explicit_references; pub mod fingerprint_storage; pub mod graph; +pub mod identifier_fitting; pub mod lineage; pub mod lineage_command; pub mod lineage_graph; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs index 3831b2b702..12aa7365ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs @@ -1,18 +1,36 @@ //! Building, intersecting, merging, splitting and capping half-open cursor intervals. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use std::cmp::Ordering; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; use pyo3::types::{PySlice, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cursor::algebra::main::cap_from_end::cap_from_end; use sqlbuild_cursor::algebra::main::cap_from_start::cap_from_start; use sqlbuild_cursor::algebra::main::clamp::clamp; +use sqlbuild_cursor::algebra::main::compare::compare; use sqlbuild_cursor::algebra::main::merge::merge; use sqlbuild_cursor::algebra::main::observed_partition::observed_partition; use sqlbuild_cursor::algebra::main::split_into_batches::split_into_batches; use sqlbuild_cursor::algebra::main::validate_interval::validate_interval; use sqlbuild_cursor::algebra::models::{Interval, Scalar}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::cursor::conversion::CursorSession; +use crate::bindings::_helpers::cursor::scalars::as_scalar; + +const MICROBATCH_MODELS_MODULE: &str = "sqlbuild.microbatches.models"; +const MICROBATCH_EXCEPTIONS_MODULE: &str = "sqlbuild.microbatches.exceptions"; +const RAW_INTERVAL_CURSOR_TYPES: [&str; 2] = ["timestamp", "integer"]; + +/// One stored interval with its parsed bounds. +struct RawInterval<'py> { + start: Scalar, + end: Scalar, + interval: Bound<'py, PyAny>, +} fn python_intervals<'py>( session: &CursorSession<'py>, @@ -84,6 +102,69 @@ fn merge_intervals<'py>(intervals: &Bound<'py, PyAny>) -> PyResult( + intervals: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = intervals.py(); + let kind: String = cursor_type.str()?.extract()?; + if !RAW_INTERVAL_CURSOR_TYPES.contains(&kind.as_str()) { + return Err( + imported(py, MICROBATCH_EXCEPTIONS_MODULE, "MicrobatchStateError") + .and_then(|class| class.call1((format!("unsupported cursor type: {kind}"),))) + .map_or_else(|error| error, PyErr::from_value), + ); + } + let session: CursorSession<'py> = CursorSession::new(py)?; + let mut parsed: Vec> = Vec::new(); + for interval in intervals.try_iter()? { + let interval: Bound<'py, PyAny> = interval?; + parsed.push(RawInterval { + start: as_scalar(&session, &interval.getattr("start")?, cursor_type)?, + end: as_scalar(&session, &interval.getattr("end")?, cursor_type)?, + interval, + }); + } + let mut failure: Option = None; + parsed.sort_by(|left, right| { + session + .checked(compare(&left.start, &right.start, &session)) + .unwrap_or_else(|error| { + failure.get_or_insert(error); + Ordering::Equal + }) + }); + if let Some(error) = failure { + return Err(error); + } + let class: Bound<'py, PyAny> = imported(py, MICROBATCH_MODELS_MODULE, "MicrobatchInterval")?; + let mut merged: Vec> = Vec::new(); + for next in parsed { + let Some(last) = merged.last_mut() else { + merged.push(next); + continue; + }; + if session.checked(compare(&last.end, &next.start, &session))? == Ordering::Less { + merged.push(next); + continue; + } + if session.checked(compare(&last.end, &next.end, &session))? == Ordering::Less { + last.interval = call_with( + &class, + &[ + ("start", last.interval.getattr("start")?), + ("end", next.interval.getattr("end")?), + ], + )?; + last.end = next.end; + } + } + PyTuple::new(py, merged.into_iter().map(|raw| raw.interval)) +} + /// An interval split into batches of a positive number of grain units. #[pyfunction(name = "cursor_split_into_batches")] #[pyo3(signature = (*, interval, step))] @@ -124,6 +205,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(partition, module)?)?; module.add_function(wrap_pyfunction!(clamp_intervals, module)?)?; module.add_function(wrap_pyfunction!(merge_intervals, module)?)?; + module.add_function(wrap_pyfunction!(merge_raw_intervals, module)?)?; module.add_function(wrap_pyfunction!(split, module)?)?; module.add_function(wrap_pyfunction!(cap_start, module)?)?; module.add_function(wrap_pyfunction!(cap_end, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs index c6582585cb..f551d426da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs @@ -170,7 +170,7 @@ pub(crate) fn parse_value<'py>( } /// `raw` unchanged when it is already a typed value, else parsed. -fn as_scalar<'py>( +pub(crate) fn as_scalar<'py>( session: &CursorSession<'py>, raw: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index ccc4eacfbb..a63b0b867b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -66,6 +66,7 @@ pub(crate) mod query_command; pub(crate) mod records; pub(crate) mod refactor; pub(crate) mod references; +pub(crate) mod relation_naming; pub(crate) mod render; pub(crate) mod result_output; pub(crate) mod rules; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs index ad4968c18c..1fcf7cf738 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs @@ -3,9 +3,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::{ - planner_changes, planner_entries, planner_identity, planner_migrations, planner_models, - planner_phases, planner_resolution, planner_scenario, planner_scope, planner_selection, - planner_strategy, planner_warehouse, + planner_changes, planner_entries, planner_identity, planner_migrations, planner_models, planner_phases, planner_resolution, planner_scenario, planner_scope, planner_selection, planner_strategy, planner_warehouse, relation_naming, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -21,5 +19,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { planner_strategy::registration::register(module)?; planner_resolution::registration::register(module)?; planner_entries::registration::register(module)?; + relation_naming::registration::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs index 4aa6985c0e..edda0220c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs @@ -8,10 +8,13 @@ use crate::bindings::_helpers::planner_scenario::constants::{ SCENARIO_PLAN_RELATION_COLLISION, }; use crate::bindings::_helpers::planner_scenario::values::{MODELS, TYPES, error, string}; +use crate::bindings::_helpers::relation_naming::fitting::fitted_logical_name; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_scenario::main::{hash_prefix, parse_artifact_name}; +const SCENARIO_ARTIFACT_LABEL: &str = "Scenario artifact"; + #[pyfunction] #[pyo3(signature = (**kwargs))] pub(crate) fn compute_scenario_hash_prefix( @@ -106,24 +109,17 @@ pub(crate) fn build_scenario_hash_index<'py>( #[pyfunction] #[pyo3(signature = (*, logical_name, fixed_prefix, identifier_limit))] -pub(crate) fn fit_scenario_artifact_logical_name<'py>( - py: Python<'py>, - logical_name: Bound<'py, PyAny>, - fixed_prefix: Bound<'py, PyAny>, - identifier_limit: Bound<'py, PyAny>, -) -> PyResult> { - call_with( - &imported( - py, - "sqlbuild.adapter.relations.main.fit_artifact_logical_name", - "fit_artifact_logical_name", - )?, - &[ - ("logical_name", logical_name), - ("fixed_prefix", fixed_prefix), - ("identifier_limit", identifier_limit), - ("artifact_label", string(py, "Scenario artifact")), - ], +pub(crate) fn fit_scenario_artifact_logical_name( + py: Python<'_>, + logical_name: &str, + fixed_prefix: &str, + identifier_limit: i64, +) -> PyResult { + fitted_logical_name( + py, + (logical_name, fixed_prefix), + identifier_limit, + SCENARIO_ARTIFACT_LABEL, ) } @@ -133,17 +129,16 @@ pub(crate) fn build_scenario_artifact_physical_name( hash_prefix: &str, kind: &str, logical_name: Bound<'_, PyAny>, - identifier_limit: Bound<'_, PyAny>, + identifier_limit: i64, ) -> PyResult { let py = logical_name.py(); let prefix = format!("__sqb_{hash_prefix}__{kind}__"); let logical: String = fit_scenario_artifact_logical_name( py, - logical_name, - string(py, &prefix), + &logical_name.extract::()?, + &prefix, identifier_limit, - )? - .extract()?; + )?; Ok(format!("{prefix}{logical}")) } @@ -160,8 +155,7 @@ pub(crate) fn build_scenario_artifact_name( .call1((kind,))? .getattr("value")? .extract()?; - let limit = pyo3::IntoPyObjectExt::into_bound_py_any(identifier_limit, py)?; - build_scenario_artifact_physical_name(hash_prefix, &kind, logical_name, limit) + build_scenario_artifact_physical_name(hash_prefix, &kind, logical_name, identifier_limit) } #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs new file mode 100644 index 0000000000..86f0264adf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs @@ -0,0 +1,63 @@ +//! Generated relation names fitted within identifier limits, raising `SharedInputError`. + +use pyo3::prelude::{Bound, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::identifier_fitting::main::fit_artifact_logical_name::fit_artifact_logical_name as fit_logical; +use sqlbuild_analysis::identifier_fitting::main::fit_auxiliary_relation_name::fit_auxiliary_relation_name as fit_auxiliary; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const SHARED_EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; + +fn shared_error(py: Python<'_>, message: String) -> PyErr { + imported(py, SHARED_EXCEPTIONS_MODULE, "SharedInputError") + .and_then(|class| class.call1((message,))) + .map_or_else(|error| error, PyErr::from_value) +} + +/// A readable logical component fitted after a fixed prefix, with a hash suffix when shortened. +#[pyfunction] +#[pyo3(signature = (**options))] +fn fit_artifact_logical_name(options: Option<&Bound<'_, PyDict>>) -> PyResult { + let request = Request::new("fit_artifact_logical_name", options)?; + fitted_logical_name( + request.py(), + ( + &request.get("logical_name")?.extract::()?, + &request.get("fixed_prefix")?.extract::()?, + ), + request.get("identifier_limit")?.extract()?, + &request.get("artifact_label")?.extract::()?, + ) +} + +/// `fit_artifact_logical_name`, raising `SharedInputError` when the name cannot fit. +pub(crate) fn fitted_logical_name( + py: Python<'_>, + names: (&str, &str), + identifier_limit: i64, + artifact_label: &str, +) -> PyResult { + fit_logical(names, identifier_limit, artifact_label) + .map_err(|message| shared_error(py, message)) +} + +/// `base_name + suffix` fitted within the limit, keeping the suffix and a base hash. +#[pyfunction] +#[pyo3(signature = (*, base_name, suffix, identifier_limit))] +fn fit_auxiliary_relation_name( + py: Python<'_>, + base_name: &str, + suffix: &str, + identifier_limit: i64, +) -> PyResult { + fit_auxiliary(base_name, suffix, identifier_limit).map_err(|message| shared_error(py, message)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(fit_artifact_logical_name, module)?)?; + module.add_function(wrap_pyfunction!(fit_auxiliary_relation_name, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/mod.rs new file mode 100644 index 0000000000..0bbf05581d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod fitting; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/registration.rs new file mode 100644 index 0000000000..7a489501cf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/registration.rs @@ -0,0 +1,9 @@ +//! Register the generated relation name fitting functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::relation_naming::fitting; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + fitting::register(module) +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 3ef8fb438d..9e471d14f6 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -331,6 +331,8 @@ from sqlbuild.lint.models import ( from sqlbuild.lint.types import ( RelationKeyRow, ) +from sqlbuild.lint.types import RelationKeyRow, RelationKeys +from sqlbuild.microbatches.models import MicrobatchInterval from sqlbuild.presentation.models import DisplayOptions from sqlbuild.presentation.types import TransientLineOwner from sqlbuild.provider.classes.container import ProviderContainer @@ -4913,6 +4915,13 @@ def scope_test_ctes( ) -> list[tuple[str | None, list[tuple[str, str]]]]: ... def omitted_ceremonial_select(sql: str, syntax: dict[str, object]) -> int | None: ... def extract_sql_tests_json(request_json: str) -> str: ... +def fit_artifact_logical_name( + *, logical_name: str, fixed_prefix: str, identifier_limit: int, artifact_label: str +) -> str: ... +def fit_auxiliary_relation_name(*, base_name: str, suffix: str, identifier_limit: int) -> str: ... +def merge_microbatch_intervals( + *, intervals: Sequence[MicrobatchInterval], cursor_type: str +) -> tuple[MicrobatchInterval, ...]: ... def render_sql_test_comparison_sql( test_entries: Sequence[SqlTestPlanEntry], *, diff --git a/src/sqlbuild/adapter/relations/_helpers/identifier_fitting.py b/src/sqlbuild/adapter/relations/_helpers/identifier_fitting.py deleted file mode 100644 index 1b17b5bb0b..0000000000 --- a/src/sqlbuild/adapter/relations/_helpers/identifier_fitting.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Fit generated relation names within identifier limits measured in UTF-8 bytes.""" - -from __future__ import annotations - -import hashlib - -from sqlbuild.adapter.relations.constants import SHORTENED_LOGICAL_NAME_HASH_LENGTH -from sqlbuild.errors.contracts.exceptions import SharedInputError - - -def fit_artifact_logical_name_impl( - *, logical_name: str, fixed_prefix: str, identifier_limit: int, artifact_label: str -) -> str: - """Fit a readable logical component with a deterministic hash suffix.""" - - max_logical_length: int = identifier_limit - _byte_length(fixed_prefix) - if max_logical_length < 1: - raise SharedInputError( - f"{artifact_label} prefix '{fixed_prefix}' does not fit within identifier " - f"limit {identifier_limit}" - ) - if _byte_length(logical_name) <= max_logical_length: - return logical_name - - suffix_length: int = SHORTENED_LOGICAL_NAME_HASH_LENGTH + 1 - if max_logical_length <= suffix_length: - raise SharedInputError( - f"{artifact_label} name for '{logical_name}' cannot fit within identifier " - f"limit {identifier_limit}" - ) - logical_hash: str = hashlib.sha256(logical_name.encode("utf-8")).hexdigest()[ - :SHORTENED_LOGICAL_NAME_HASH_LENGTH - ] - prefix_length: int = max_logical_length - suffix_length - return f"{_truncate_bytes(value=logical_name, max_bytes=prefix_length)}_{logical_hash}" - - -def fit_auxiliary_relation_name_impl(*, base_name: str, suffix: str, identifier_limit: int) -> str: - """Append a suffix, shortening the base with a stable hash when the whole name is too long.""" - - if _byte_length(base_name) + _byte_length(suffix) <= identifier_limit: - return f"{base_name}{suffix}" - hash_part: str = ( - "_" - + hashlib.sha256(base_name.encode("utf-8")).hexdigest()[:SHORTENED_LOGICAL_NAME_HASH_LENGTH] - ) - kept_length: int = identifier_limit - _byte_length(suffix) - len(hash_part) - if kept_length < 1: - raise SharedInputError( - f"auxiliary relation suffix '{suffix}' for '{base_name}' cannot fit within " - f"identifier limit {identifier_limit}" - ) - return f"{_truncate_bytes(value=base_name, max_bytes=kept_length)}{hash_part}{suffix}" - - -def _byte_length(value: str) -> int: - return len(value.encode("utf-8")) - - -def _truncate_bytes(*, value: str, max_bytes: int) -> str: - """Keep the longest prefix whose UTF-8 encoding fits, never splitting a character.""" - - return value.encode("utf-8")[:max_bytes].decode("utf-8", errors="ignore") diff --git a/src/sqlbuild/adapter/relations/constants.py b/src/sqlbuild/adapter/relations/constants.py index 53d4ad02c5..e68a5951b2 100644 --- a/src/sqlbuild/adapter/relations/constants.py +++ b/src/sqlbuild/adapter/relations/constants.py @@ -3,7 +3,6 @@ INSPECTION_DESCRIBED_NAME_LIMIT: int = 3 INSPECTION_IN_LIST_LIMIT: int = 50 METADATA_NAME_FILTER_LIMIT: int = 250 -SHORTENED_LOGICAL_NAME_HASH_LENGTH: int = 8 STATEMENT_SEPARATOR: str = ";" QUOTED_IDENTIFIER_MIN_LENGTH: int = 2 DOUBLE_QUOTE: str = '"' diff --git a/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name.py b/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name.py deleted file mode 100644 index 2c601d9173..0000000000 --- a/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Generated relation name fitting entrypoint.""" - -from __future__ import annotations - -from sqlbuild.adapter.relations._helpers.identifier_fitting import fit_artifact_logical_name_impl - - -def fit_artifact_logical_name( - *, logical_name: str, fixed_prefix: str, identifier_limit: int, artifact_label: str -) -> str: - """Fit a readable logical component after a fixed prefix within an identifier limit.""" - - return fit_artifact_logical_name_impl( - logical_name=logical_name, - fixed_prefix=fixed_prefix, - identifier_limit=identifier_limit, - artifact_label=artifact_label, - ) diff --git a/src/sqlbuild/adapter/relations/main/fit_auxiliary_relation_name.py b/src/sqlbuild/adapter/relations/main/fit_auxiliary_relation_name.py deleted file mode 100644 index ec4f1a6dba..0000000000 --- a/src/sqlbuild/adapter/relations/main/fit_auxiliary_relation_name.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Auxiliary relation name fitting entrypoint.""" - -from __future__ import annotations - -from sqlbuild.adapter.relations._helpers.identifier_fitting import ( - fit_auxiliary_relation_name_impl, -) - - -def fit_auxiliary_relation_name(*, base_name: str, suffix: str, identifier_limit: int) -> str: - """Return ``base_name + suffix`` fitted within the limit, keeping the suffix and a base hash.""" - - return fit_auxiliary_relation_name_impl( - base_name=base_name, suffix=suffix, identifier_limit=identifier_limit - ) diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_names.py b/src/sqlbuild/executor/janitor/_helpers/archive_names.py index af806b6cdb..31885e515f 100644 --- a/src/sqlbuild/executor/janitor/_helpers/archive_names.py +++ b/src/sqlbuild/executor/janitor/_helpers/archive_names.py @@ -5,7 +5,7 @@ import re from datetime import UTC, datetime -from sqlbuild.adapter.relations.main.fit_artifact_logical_name import fit_artifact_logical_name +from sqlbuild._native import fit_artifact_logical_name from sqlbuild.executor.janitor.constants import ( ARCHIVE_ARTIFACT_LABEL, ARCHIVE_LOOKALIKE_PREFIX, diff --git a/src/sqlbuild/executor/load/_helpers/staging.py b/src/sqlbuild/executor/load/_helpers/staging.py index 4e2308ee91..237ae5b80b 100644 --- a/src/sqlbuild/executor/load/_helpers/staging.py +++ b/src/sqlbuild/executor/load/_helpers/staging.py @@ -4,10 +4,10 @@ from typing import Any +from sqlbuild._native import fit_auxiliary_relation_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.types import LoaderLogicalType -from sqlbuild.adapter.relations.main.fit_auxiliary_relation_name import fit_auxiliary_relation_name from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) diff --git a/src/sqlbuild/executor/run/_helpers/execution/table_targets.py b/src/sqlbuild/executor/run/_helpers/execution/table_targets.py index cb2a03f5f9..f93a559bb0 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/table_targets.py +++ b/src/sqlbuild/executor/run/_helpers/execution/table_targets.py @@ -2,8 +2,8 @@ from __future__ import annotations +from sqlbuild._native import fit_auxiliary_relation_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.relations.main.fit_auxiliary_relation_name import fit_auxiliary_relation_name from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index 9b0d288808..cf4c517f67 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -6,11 +6,14 @@ from typing import Any from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild._native import future_cursor_cap_warning, maximum_start_cap_warning +from sqlbuild._native import ( + fit_auxiliary_relation_name, + future_cursor_cap_warning, + maximum_start_cap_warning, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.adapter.relations.main.fit_auxiliary_relation_name import fit_auxiliary_relation_name from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index 2fdb534834..599fc8b238 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -16,9 +16,11 @@ compute_query_hash, cursor_bound_display, empty_cursor_window, + fit_auxiliary_relation_name, future_cursor_cap_warning, inclusive_cursor_end, maximum_start_cap_warning, + merge_microbatch_intervals, microbatch_limit_warning, resolve_effective_microbatch_batch_size, ) @@ -38,7 +40,6 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists -from sqlbuild.adapter.relations.main.fit_auxiliary_relation_name import fit_auxiliary_relation_name from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) @@ -130,7 +131,6 @@ from sqlbuild.microbatches.main.latest_replay_requirement import ( latest_active_replay_requirement, ) -from sqlbuild.microbatches.main.merge_intervals import merge_intervals from sqlbuild.microbatches.main.project_coverage import project_microbatch_coverage from sqlbuild.microbatches.main.project_replay import project_replay_requirement from sqlbuild.microbatches.models import ( @@ -415,7 +415,7 @@ def execute_microbatch_entry( if isinstance(reconciliation, ModelExecutionResult): return reconciliation state, history_context, reconciled_batches = reconciliation - resolved_intervals: tuple[MicrobatchInterval, ...] = merge_intervals( + resolved_intervals: tuple[MicrobatchInterval, ...] = merge_microbatch_intervals( intervals=tuple( MicrobatchInterval(start=render(value=batch.start), end=render(value=batch.end)) for batch in reconciled_batches @@ -1234,7 +1234,7 @@ def _microbatch_lifecycle( def _microbatch_lifecycle_plan( *, batch_plan: _MicrobatchPlan, cursor_type: str ) -> _MicrobatchLifecyclePlan: - selected_intervals: tuple[MicrobatchInterval, ...] = merge_intervals( + selected_intervals: tuple[MicrobatchInterval, ...] = merge_microbatch_intervals( intervals=tuple( MicrobatchInterval(start=render(value=batch.start), end=render(value=batch.end)) for batch in batch_plan.batches @@ -3296,7 +3296,7 @@ def _plan_microbatch_windows( batch_size=batch_size, effective_grain=effective_grain, ) - work_intervals: tuple[MicrobatchInterval, ...] = merge_intervals( + work_intervals: tuple[MicrobatchInterval, ...] = merge_microbatch_intervals( intervals=( ( MicrobatchInterval( diff --git a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py index 911926fe1f..9c15fb0b75 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import fit_auxiliary_relation_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( @@ -13,7 +14,6 @@ ) from sqlbuild.adapter.contract.types import TypeFamily from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists -from sqlbuild.adapter.relations.main.fit_auxiliary_relation_name import fit_auxiliary_relation_name from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) diff --git a/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py b/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py index bf0cf58fa7..e8863dcb82 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py +++ b/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py @@ -4,10 +4,10 @@ from typing import Any +from sqlbuild._native import fit_auxiliary_relation_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.adapter.relations.main.fit_auxiliary_relation_name import fit_auxiliary_relation_name from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) diff --git a/src/sqlbuild/microbatches/main/merge_intervals.py b/src/sqlbuild/microbatches/main/merge_intervals.py deleted file mode 100644 index 2040611c91..0000000000 --- a/src/sqlbuild/microbatches/main/merge_intervals.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Merge canonical microbatch intervals.""" - -from sqlbuild._native import cursor_compare as compare -from sqlbuild._native import cursor_merge as merge -from sqlbuild._native import cursor_parse as parse -from sqlbuild._native import cursor_render as render -from sqlbuild._native import cursor_sort_key -from sqlbuild.compiler.planner.types import CursorType -from sqlbuild.cursor_algebra.models import AlignedInterval, IntegerValue -from sqlbuild.cursor_algebra.types import CursorScalar -from sqlbuild.microbatches.exceptions import MicrobatchStateError -from sqlbuild.microbatches.models import MicrobatchInterval - - -def merge_intervals( - *, intervals: tuple[MicrobatchInterval, ...], cursor_type: str -) -> tuple[MicrobatchInterval, ...]: - """Merge overlapping or adjacent intervals while preserving disjoint sets.""" - - if cursor_type not in {CursorType.TIMESTAMP, CursorType.INTEGER}: - raise MicrobatchStateError(f"unsupported cursor type: {cursor_type}") - if cursor_type == CursorType.INTEGER: - typed_merged: tuple[MicrobatchInterval, ...] | None = _merge_typed_integers( - intervals=intervals - ) - if typed_merged is not None: - return typed_merged - ordered: tuple[MicrobatchInterval, ...] = tuple( - sorted( - intervals, - key=lambda interval: cursor_sort_key(raw=interval.start, cursor_type=cursor_type), - ) - ) - merged: list[MicrobatchInterval] = [] - for interval in ordered: - if not merged or _lt(left=merged[-1].end, right=interval.start, cursor_type=cursor_type): - merged.append(interval) - continue - if _lt(left=merged[-1].end, right=interval.end, cursor_type=cursor_type): - merged[-1] = MicrobatchInterval(start=merged[-1].start, end=interval.end) - return tuple(merged) - - -def _lt(*, left: str, right: str, cursor_type: str) -> bool: - return ( - compare( - left=parse(raw=left, cursor_type=cursor_type), - right=parse(raw=right, cursor_type=cursor_type), - ) - < 0 - ) - - -def _merge_typed_integers( - *, intervals: tuple[MicrobatchInterval, ...] -) -> tuple[MicrobatchInterval, ...] | None: - typed: list[AlignedInterval] = [] - for interval in intervals: - start: CursorScalar = parse(raw=interval.start, cursor_type=CursorType.INTEGER) - end: CursorScalar = parse(raw=interval.end, cursor_type=CursorType.INTEGER) - if ( - not isinstance(start, IntegerValue) - or not isinstance(end, IntegerValue) - or render(value=start) != interval.start - or render(value=end) != interval.end - or start.value >= end.value - ): - return None - typed.append(AlignedInterval(start=start, end=end, grain=None)) - return tuple( - MicrobatchInterval(start=render(value=interval.start), end=render(value=interval.end)) - for interval in merge(intervals=tuple(typed)) - ) diff --git a/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/test_fit_artifact_logical_name.py b/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/test_fit_artifact_logical_name.py index 40c3632495..55e52285af 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/test_fit_artifact_logical_name.py +++ b/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/test_fit_artifact_logical_name.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.adapter.relations.main.fit_artifact_logical_name import fit_artifact_logical_name +from sqlbuild._native import fit_artifact_logical_name from sqlbuild.errors.contracts.exceptions import SharedInputError from tests.unit.src.sqlbuild.adapter.relations.main.fit_artifact_logical_name._test_types import ( FitArtifactLogicalNameErrorTestCase, diff --git a/tests/unit/src/sqlbuild/adapter/relations/main/fit_auxiliary_relation_name/test_fit_auxiliary_relation_name.py b/tests/unit/src/sqlbuild/adapter/relations/main/fit_auxiliary_relation_name/test_fit_auxiliary_relation_name.py index c70d7e4456..16da3b3877 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/main/fit_auxiliary_relation_name/test_fit_auxiliary_relation_name.py +++ b/tests/unit/src/sqlbuild/adapter/relations/main/fit_auxiliary_relation_name/test_fit_auxiliary_relation_name.py @@ -4,9 +4,7 @@ import pytest -from sqlbuild.adapter.relations.main.fit_auxiliary_relation_name import ( - fit_auxiliary_relation_name, -) +from sqlbuild._native import fit_auxiliary_relation_name from sqlbuild.errors.contracts.exceptions import SharedInputError from tests.unit.src.sqlbuild.adapter.relations.main.fit_auxiliary_relation_name._test_types import ( FitAuxiliaryRelationNameErrorTestCase, From 7f6b6b7168b2e674577d48258032a8b6293528f4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:46:24 +0100 Subject: [PATCH 03/11] refactor: resolve diff tolerances and check raw diff queries natively Raw-query diff validation parses with the native SQL library instead of the Python binding of the same library, and row-diff tolerance and sampling configuration is resolved by native functions shared by the CLI and the executor. --- .../src/diff_selection/constants.rs | 3 + .../src/diff_selection/main/mod.rs | 1 + .../diff_selection/main/raw_query_shape.rs | 45 +++ .../src/diff_selection/models.rs | 10 + .../src/bindings/_helpers/diff_report/mod.rs | 1 + .../bindings/_helpers/diff_report/query.rs | 95 ++--- .../_helpers/diff_report/registration.rs | 4 +- .../_helpers/diff_report/tolerances.rs | 349 ++++++++++++++++++ src/sqlbuild/_native.pyi | 14 +- src/sqlbuild/executor/diff/_helpers/config.py | 177 --------- .../executor/diff/_helpers/execution.py | 10 +- src/sqlbuild/executor/diff/constants.py | 5 - src/sqlbuild/executor/diff/main/_config.py | 12 - .../src/sqlbuild/executor/diff/test_config.py | 6 +- 14 files changed, 459 insertions(+), 273 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs delete mode 100644 src/sqlbuild/executor/diff/_helpers/config.py delete mode 100644 src/sqlbuild/executor/diff/main/_config.py diff --git a/crates/sqlbuild-analysis/src/diff_selection/constants.rs b/crates/sqlbuild-analysis/src/diff_selection/constants.rs index e85095e908..13daec5375 100644 --- a/crates/sqlbuild-analysis/src/diff_selection/constants.rs +++ b/crates/sqlbuild-analysis/src/diff_selection/constants.rs @@ -11,3 +11,6 @@ pub(crate) const EMPTY_SELECTOR_CODE: &str = "S001"; pub(crate) const UNKNOWN_NAME_CODE: &str = "S007"; pub(crate) const UNKNOWN_TAG_CODE: &str = "S008"; pub(crate) const UNKNOWN_PATH_CODE: &str = "S009"; +pub(crate) const RAW_QUERY_READ_ONLY_ROOTS: [&str; 5] = + ["except", "intersect", "select", "union", "values"]; +pub(crate) const RAW_QUERY_GUARD: &str = r#"{"maxFunctionCallDepth": 512}"#; diff --git a/crates/sqlbuild-analysis/src/diff_selection/main/mod.rs b/crates/sqlbuild-analysis/src/diff_selection/main/mod.rs index 5f8a0fa05f..82abb2ca13 100644 --- a/crates/sqlbuild-analysis/src/diff_selection/main/mod.rs +++ b/crates/sqlbuild-analysis/src/diff_selection/main/mod.rs @@ -1 +1,2 @@ +pub mod raw_query_shape; pub mod resolve_diff_selection; diff --git a/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs b/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs new file mode 100644 index 0000000000..c553e213e1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs @@ -0,0 +1,45 @@ +//! Parse one raw diff query as the Polyglot wheel's guarded `parse_one` does and read its shape. + +use polyglot_sql::dialects::Dialect; +use polyglot_sql::traversal::ExpressionWalk; +use polyglot_sql::{ComplexityGuardOptions, Expression, ParseOptions}; + +use crate::diff_selection::constants::{RAW_QUERY_GUARD, RAW_QUERY_READ_ONLY_ROOTS}; +use crate::diff_selection::models::RawQueryShape; + +/// The query's shape, or the text of the exception the wheel raises for it. +pub fn raw_query_shape(sql: &str, dialect: &str) -> Result { + let parser: Dialect = + Dialect::get_by_name(dialect).ok_or_else(|| format!("Unknown dialect: {dialect}"))?; + let guard: ComplexityGuardOptions = + serde_json::from_str(RAW_QUERY_GUARD).map_err(|error| error.to_string())?; + let mut statements: Vec = parser + .parse_with_options( + sql, + &ParseOptions { + complexity_guard: Some(guard), + }, + ) + .map_err(|error| error.to_string())?; + if statements.len() != 1 { + return Err(format!("Expected 1 statement, found {}", statements.len())); + } + let root: Expression = statements.remove(0); + let root_key: String = root.variant_name().to_lowercase(); + let node_keys: Vec = if RAW_QUERY_READ_ONLY_ROOTS.contains(&root_key.as_str()) { + root.dfs() + .map(|node| node.variant_name().to_lowercase()) + .collect() + } else { + Vec::new() + }; + let writes_or_locks: bool = match &root { + Expression::Select(select) => select.into.is_some() || !select.locks.is_empty(), + _ => false, + }; + Ok(RawQueryShape { + root_key, + node_keys, + writes_or_locks, + }) +} diff --git a/crates/sqlbuild-analysis/src/diff_selection/models.rs b/crates/sqlbuild-analysis/src/diff_selection/models.rs index d95cbdf0dd..f20f588539 100644 --- a/crates/sqlbuild-analysis/src/diff_selection/models.rs +++ b/crates/sqlbuild-analysis/src/diff_selection/models.rs @@ -14,3 +14,13 @@ pub struct DiffGraph { /// `(resource, model folder)`. pub paths: Vec<(usize, String)>, } + +/// The parsed shape of one raw diff query: lowercased expression kinds and write clauses. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RawQueryShape { + pub root_key: String, + /// Every node kind in depth-first order, collected only for read-only roots. + pub node_keys: Vec, + /// The root `SELECT` has an `INTO` target or row-locking clauses. + pub writes_or_locks: bool, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs index a2dea05e2a..d0a3f88ba1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/mod.rs @@ -3,3 +3,4 @@ pub(crate) mod facts; pub(crate) mod query; pub(crate) mod registration; pub(crate) mod support; +pub(crate) mod tolerances; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index 420f59afef..df0ee54d16 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -12,24 +12,28 @@ use sqlbuild_cli::diff_report::main::read_only_query_problem::read_only_query_pr use sqlbuild_cli::diff_report::models::QueryShapeFacts; use sqlbuild_cli::diff_report::types::PhaseSeconds; +use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::diff_report::support::{ call, cli_error, error_text, none, object, progress_line, }; +use crate::bindings::_helpers::diff_report::tolerances::cli_tolerance_overrides; +use crate::bindings::types::CompilerDetach; +use sqlbuild_analysis::diff_selection::main::raw_query_shape::raw_query_shape; +use sqlbuild_analysis::diff_selection::models::RawQueryShape; const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const NAMESPACE_MODULE: &str = "sqlbuild._native"; const RUNTIME_MODULE: &str = "sqlbuild._native"; -const POLYGLOT_MODULE: &str = "sqlbuild.compiler.sql_analysis.main.import_polyglot_sql"; +const GENERIC_DIALECT: &str = "generic"; const COMMAND_MODELS_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.executor.diff.classes.query_artifact_lifecycle"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; const CURSOR_MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; const EXECUTE_QUERY_MODULE: &str = "sqlbuild.executor.diff.main._execute_query"; const DIFF_MODELS_MODULE: &str = "sqlbuild.executor.diff.models"; -const DIFF_CONFIG_MODULE: &str = "sqlbuild.executor.diff.main._config"; const NAMESPACELESS_ADAPTERS: [&str; 2] = ["duckdb", "motherduck"]; fn failure(py: Python<'_>, message: String, code: &str, help: Option<&str>) -> PyErr { @@ -114,13 +118,6 @@ fn query_input( } } -fn node_key(node: &Bound<'_, PyAny>) -> PyResult { - match node.getattr_opt("key")? { - Some(key) => Ok(key.str()?.to_string().to_lowercase()), - None => Ok(String::new()), - } -} - fn read_only_query( py: Python<'_>, sql: &str, @@ -139,69 +136,29 @@ fn read_only_query( None, )); } - let polyglot: Bound<'_, PyAny> = - imported(py, POLYGLOT_MODULE, "import_polyglot_sql")?.call0()?; - if polyglot.is_none() { - return Err(failure( - py, - "raw-query diff requires the bundled SQL parser".to_owned(), - "C233", - None, - )); - } let dialect: Bound<'_, PyAny> = adapter.call_method0("sql_analysis_dialect")?; - let dialect: Bound<'_, PyAny> = if dialect.is_truthy()? { - dialect + let dialect: String = if dialect.is_truthy()? { + dialect.extract()? } else { - object(py, "generic")? - }; - let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("dialect", dialect)?; - let expression: Bound<'_, PyAny> = - match polyglot.call_method("parse_one", (normalized,), Some(&keywords)) { - Ok(expression) => expression, - Err(error) if error.is_instance_of::(py) => { - let replacement: PyErr = failure( - py, - format!( - "{label} query is not one valid SQL statement: {}", - error_text(py, &error)? - ), - "C234", - None, - ); - replacement.set_cause(py, Some(error)); - return Err(replacement); - } - Err(error) => return Err(error), - }; - let mut node_keys: Vec = Vec::new(); - let root_key: String = node_key(&expression)?; - let read_only_root: bool = - ["except", "intersect", "select", "union", "values"].contains(&root_key.as_str()); - if read_only_root { - for node in expression.call_method0("walk")?.try_iter()? { - node_keys.push(node_key(&node?)?); - } - } - let arguments: Bound<'_, PyAny> = expression.getattr_opt("args")?.unwrap_or_else(|| none(py)); - let writes_or_locks: bool = match arguments.downcast::() { - Ok(arguments) => { - arguments - .get_item("into")? - .is_some_and(|into| !into.is_none()) - || arguments - .get_item("locks")? - .map_or(Ok(false), |locks| locks.is_truthy())? - } - Err(_) => false, + GENERIC_DIALECT.to_owned() }; + let shape: RawQueryShape = py + .compiler_detach(|| Ok::<_, String>(raw_query_shape(normalized, &dialect))) + .map_err(value_error)? + .map_err(|message| { + failure( + py, + format!("{label} query is not one valid SQL statement: {message}"), + "C234", + None, + ) + })?; read_only_query_problem( label, &QueryShapeFacts { - root_key, - node_keys, - writes_or_locks, + root_key: shape.root_key, + node_keys: shape.node_keys, + writes_or_locks: shape.writes_or_locks, }, ) .map_err(|error| user_failure(py, error))?; @@ -358,9 +315,9 @@ pub(crate) fn execution_options<'py>( ("exhaustive", &attribute("exhaustive")?), ], )?; - let tolerances: Bound<'py, PyAny> = call( - &imported(py, DIFF_CONFIG_MODULE, "parse_cli_tolerance_overrides")?, - &[("values", &attribute("tolerance_overrides")?)], + let tolerances: Bound<'py, PyAny> = cli_tolerance_overrides( + py, + &attribute("tolerance_overrides")?.extract::>()?, )?; let schema_only: Bound<'py, PyAny> = attribute("schema_only")?; let collect: Bound<'py, PyAny> = object(py, !schema_only.is_truthy()?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs index 78fa537525..6f861c78b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs @@ -17,6 +17,7 @@ use crate::bindings::_helpers::diff_report::command::{ execute_prepared_query_diff, run_diff_command, }; use crate::bindings::_helpers::diff_report::facts::{diff_facts, json_of}; +use crate::bindings::_helpers::diff_report::tolerances; use crate::bindings::_helpers::frontier::encoder::emit_error; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use sqlbuild_cli::diff_report::types::PhaseSeconds; @@ -216,5 +217,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(render_diff_output, module)?)?; module.add_function(wrap_pyfunction!(render_diff_error_json, module)?)?; module.add_function(wrap_pyfunction!(run_diff_command, module)?)?; - module.add_function(wrap_pyfunction!(execute_prepared_query_diff, module)?) + module.add_function(wrap_pyfunction!(execute_prepared_query_diff, module)?)?; + tolerances::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs new file mode 100644 index 0000000000..308f7960de --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs @@ -0,0 +1,349 @@ +//! Row-diff tolerance and sampling configuration from model config and CLI overrides. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyString}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; +const EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; +const DECIMAL_MODULE: &str = "decimal"; +const ABSOLUTE_KEY: &str = "absolute"; +const RELATIVE_KEY: &str = "relative"; +const TOLERANCE_KEYS: [&str; 2] = [ABSOLUTE_KEY, RELATIVE_KEY]; +const DEFAULT_TOLERANCES_LABEL: &str = "row_diff_tolerances"; +const CLI_TOLERANCE_MESSAGE: &str = + "diff --tolerance must use COLUMN:absolute=VALUE or COLUMN:relative=VALUE"; + +/// `ExecutorInputError(message, code=code)`. +fn input_error(py: Python<'_>, message: String, code: &str) -> PyErr { + imported(py, EXCEPTIONS_MODULE, "ExecutorInputError") + .and_then(|class| { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("code", code)?; + class.call((message,), Some(&keywords)) + }) + .map_or_else(|error| error, PyErr::from_value) +} + +/// `RowDiffTolerances` parsed from raw model configuration. +#[pyfunction] +#[pyo3(signature = (*, raw, label = DEFAULT_TOLERANCES_LABEL))] +fn parse_row_diff_tolerances<'py>( + raw: Bound<'py, PyAny>, + label: &str, +) -> PyResult> { + parsed_tolerances(&raw, label) +} + +fn parsed_tolerances<'py>(raw: &Bound<'py, PyAny>, label: &str) -> PyResult> { + let py: Python<'py> = raw.py(); + let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "RowDiffTolerances")?; + if raw.is_none() { + return class.call0(); + } + let Ok(mapping) = raw.downcast::() else { + return Err(input_error( + py, + format!("{label} must be a mapping"), + "X401", + )); + }; + let by_type: Bound<'py, PyDict> = tolerance_section( + py, + mapping.get_item("by_type")?, + (&format!("{label}.by_type"), true), + )?; + let by_column: Bound<'py, PyDict> = tolerance_section( + py, + mapping.get_item("by_column")?, + (&format!("{label}.by_column"), false), + )?; + call_with( + &class, + &[ + ("by_type", by_type.into_any()), + ("by_column", by_column.into_any()), + ], + ) +} + +fn tolerance_section<'py>( + py: Python<'py>, + raw: Option>, + (label, normalize_key): (&str, bool), +) -> PyResult> { + let parsed: Bound<'py, PyDict> = PyDict::new(py); + let Some(raw) = raw.filter(|raw| !raw.is_none()) else { + return Ok(parsed); + }; + let Ok(mapping) = raw.downcast::() else { + return Err(input_error( + py, + format!("{label} must be a mapping"), + "X401", + )); + }; + for (key, rule) in mapping.iter() { + let Ok(text) = key.downcast::() else { + return Err(input_error( + py, + format!("{label} keys must be non-empty strings"), + "X402", + )); + }; + let key_text: String = text.extract()?; + if key_text.is_empty() { + return Err(input_error( + py, + format!("{label} keys must be non-empty strings"), + "X402", + )); + } + let parsed_key: Bound<'py, PyAny> = if normalize_key { + key.call_method0("lower")? + } else { + key.clone() + }; + parsed.set_item( + parsed_key, + tolerance_rule(&rule, &format!("{label}.{key_text}"))?, + )?; + } + Ok(parsed) +} + +fn tolerance_rule<'py>(raw: &Bound<'py, PyAny>, label: &str) -> PyResult> { + let py: Python<'py> = raw.py(); + let Ok(mapping) = raw.downcast::() else { + return Err(input_error( + py, + format!("{label} must be a mapping"), + "X401", + )); + }; + let mut unsupported: Vec = Vec::new(); + for key in mapping.keys() { + let known: bool = key + .extract::() + .is_ok_and(|text| TOLERANCE_KEYS.contains(&text.as_str())); + if !known { + unsupported.push(key.str()?.extract()?); + } + } + if !unsupported.is_empty() { + return Err(input_error( + py, + format!( + "{label} contains unsupported keys: {}", + unsupported.join(", ") + ), + "X403", + )); + } + let absolute: Option> = optional_decimal( + mapping.get_item(ABSOLUTE_KEY)?, + &format!("{label}.absolute"), + )?; + let relative: Option> = optional_decimal( + mapping.get_item(RELATIVE_KEY)?, + &format!("{label}.relative"), + )?; + if absolute.is_none() && relative.is_none() { + return Err(input_error( + py, + format!("{label} must define absolute or relative"), + "X404", + )); + } + call_with( + &imported(py, MODELS_MODULE, "RowDiffTolerance")?, + &[ + ( + "absolute", + absolute.unwrap_or_else(|| py.None().into_bound(py)), + ), + ( + "relative", + relative.unwrap_or_else(|| py.None().into_bound(py)), + ), + ], + ) +} + +fn optional_decimal<'py>( + raw: Option>, + label: &str, +) -> PyResult>> { + let Some(raw) = raw.filter(|raw| !raw.is_none()) else { + return Ok(None); + }; + let py: Python<'py> = raw.py(); + let decimal_module: Bound<'py, PyModule> = py.import(DECIMAL_MODULE)?; + let decimal_class: Bound<'py, PyAny> = decimal_module.getattr("Decimal")?; + let numeric: bool = !raw.is_instance_of::() + && (raw.is_instance_of::() + || raw.is_instance_of::() + || raw.is_instance_of::() + || raw.is_instance(&decimal_class)?); + if !numeric { + return Err(input_error(py, format!("{label} must be numeric"), "X405")); + } + match decimal_class.call1((raw.str()?,)) { + Ok(value) => Ok(Some(value)), + Err(error) if error.is_instance(py, &decimal_module.getattr("InvalidOperation")?) => { + let replacement: PyErr = input_error(py, format!("{label} must be numeric"), "X405"); + replacement.set_cause(py, Some(error)); + Err(replacement) + } + Err(error) => Err(error), + } +} + +fn optional_integer<'py>(raw: &Bound<'py, PyAny>, label: &str) -> PyResult> { + if raw.is_none() { + return Ok(None); + } + if raw.is_instance_of::() || !raw.is_instance_of::() { + return Err(input_error( + raw.py(), + format!("{label} must be an integer"), + "X406", + )); + } + raw.extract().map(Some) +} + +/// `RowDiffTolerances` from repeated `COLUMN:KIND=VALUE` CLI specifications. +#[pyfunction] +#[pyo3(signature = (*, values))] +fn parse_cli_tolerance_overrides<'py>( + py: Python<'py>, + values: Vec, +) -> PyResult> { + cli_tolerance_overrides(py, &values) +} + +/// `parse_cli_tolerance_overrides` for native callers. +pub(crate) fn cli_tolerance_overrides<'py>( + py: Python<'py>, + values: &[String], +) -> PyResult> { + let by_column: Bound<'py, PyDict> = PyDict::new(py); + for value in values { + let (column_and_kind, amount) = value.split_once('=').unwrap_or((value, "")); + let separator: bool = value.contains('='); + let (column, kind) = column_and_kind + .split_once(':') + .map_or((column_and_kind, None), |(column, kind)| { + (column, Some(kind)) + }); + let valid_kind: Option<&str> = kind.filter(|kind| TOLERANCE_KEYS.contains(kind)); + let (Some(kind), true, false, false) = + (valid_kind, separator, column.is_empty(), amount.is_empty()) + else { + return Err(input_error(py, CLI_TOLERANCE_MESSAGE.to_owned(), "X408")); + }; + let rule: Bound<'py, PyDict> = match by_column.get_item(column)? { + Some(rule) => rule.downcast_into()?, + None => { + let rule: Bound<'py, PyDict> = PyDict::new(py); + by_column.set_item(column, &rule)?; + rule + } + }; + rule.set_item(kind, amount)?; + } + let raw: Bound<'py, PyDict> = PyDict::new(py); + raw.set_item("by_column", by_column)?; + parsed_tolerances(raw.as_any(), DEFAULT_TOLERANCES_LABEL) +} + +/// Model sampling configuration resolved with invocation overrides, or `None`. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_row_diff_sampling<'py>( + options: Option<&Bound<'py, PyDict>>, +) -> PyResult>> { + let request = Request::new("resolve_row_diff_sampling", options)?; + let py: Python<'py> = request.py(); + let override_values: Bound<'py, PyAny> = request.get("override")?; + if override_values.getattr("exhaustive")?.is_truthy()? { + return Ok(None); + } + let label: String = request.get("label")?.extract()?; + let overridden = |field: &str, raw: &str| -> PyResult> { + let value: Bound<'py, PyAny> = override_values.getattr(field)?; + if value.is_none() { + request.get(raw) + } else { + Ok(value) + } + }; + let row_limit_label: String = format!("{label}.row_diff_sample_rows"); + let row_limit: Option = + optional_integer(&overridden("row_limit", "raw_row_limit")?, &row_limit_label)?; + if row_limit.is_some_and(|value| value < 0) { + return Err(input_error( + py, + format!("{row_limit_label} must be zero or greater"), + "X407", + )); + } + let Some(row_limit) = row_limit.filter(|value| *value != 0) else { + return Ok(None); + }; + let seed: Option = optional_integer( + &overridden("seed", "raw_seed")?, + &format!("{label}.row_diff_sample_seed"), + )?; + call_with( + &imported(py, MODELS_MODULE, "RowDiffSampling")?, + &[ + ("row_limit", row_limit.into_bound_py_any(py)?), + ("seed", seed.unwrap_or(0).into_bound_py_any(py)?), + ], + ) + .map(Some) +} + +/// Invocation tolerance rules overlaid on compiled model rules. +#[pyfunction] +#[pyo3(signature = (**options))] +fn merge_row_diff_tolerances<'py>( + options: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("merge_row_diff_tolerances", options)?; + let py: Python<'py> = request.py(); + let base: Bound<'py, PyAny> = request.get("base")?; + let overlay: Bound<'py, PyAny> = request.get("override")?; + if overlay.is_none() { + return Ok(base); + } + let merged = |field: &str| -> PyResult> { + let combined: Bound<'py, PyDict> = PyDict::new(py); + combined.update(base.getattr(field)?.downcast()?.as_mapping())?; + combined.update(overlay.getattr(field)?.downcast()?.as_mapping())?; + Ok(combined.into_any()) + }; + call_with( + &imported(py, MODELS_MODULE, "RowDiffTolerances")?, + &[ + ("by_type", merged("by_type")?), + ("by_column", merged("by_column")?), + ("sampling", base.getattr("sampling")?), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(parse_row_diff_tolerances, module)?)?; + module.add_function(wrap_pyfunction!(parse_cli_tolerance_overrides, module)?)?; + module.add_function(wrap_pyfunction!(resolve_row_diff_sampling, module)?)?; + module.add_function(wrap_pyfunction!(merge_row_diff_tolerances, module)?)?; + Ok(()) +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 9e471d14f6..1540a6f7c5 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -20,6 +20,8 @@ from sqlbuild.adapter.contract.models import ( ExpressionInferenceProfile, QueryResult, RelationInfo, + RowDiffSampling, + RowDiffTolerances, ) from sqlbuild.adapter.contract.types import ( AdapterExecute, @@ -297,7 +299,7 @@ from sqlbuild.executor.build.models import ( SeedExecutionResult, ) from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult -from sqlbuild.executor.diff.models import DiffExecutionResult +from sqlbuild.executor.diff.models import DiffExecutionResult, RowDiffSamplingOverride from sqlbuild.executor.janitor.models import JanitorExecutionResult, JanitorPlan from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.python_nodes.models import ( @@ -4915,6 +4917,16 @@ def scope_test_ctes( ) -> list[tuple[str | None, list[tuple[str, str]]]]: ... def omitted_ceremonial_select(sql: str, syntax: dict[str, object]) -> int | None: ... def extract_sql_tests_json(request_json: str) -> str: ... +def parse_row_diff_tolerances( + *, raw: object, label: str = "row_diff_tolerances" +) -> RowDiffTolerances: ... +def parse_cli_tolerance_overrides(*, values: Sequence[str]) -> RowDiffTolerances: ... +def resolve_row_diff_sampling( + *, raw_row_limit: object, raw_seed: object, override: RowDiffSamplingOverride, label: str +) -> RowDiffSampling | None: ... +def merge_row_diff_tolerances( + *, base: RowDiffTolerances, override: RowDiffTolerances | None +) -> RowDiffTolerances: ... def fit_artifact_logical_name( *, logical_name: str, fixed_prefix: str, identifier_limit: int, artifact_label: str ) -> str: ... diff --git a/src/sqlbuild/executor/diff/_helpers/config.py b/src/sqlbuild/executor/diff/_helpers/config.py deleted file mode 100644 index 0ec3179925..0000000000 --- a/src/sqlbuild/executor/diff/_helpers/config.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Configuration helpers for diff execution.""" - -from __future__ import annotations - -from decimal import Decimal, InvalidOperation -from typing import cast - -from sqlbuild.adapter.contract.models import RowDiffSampling, RowDiffTolerance, RowDiffTolerances -from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.executor.diff.constants import ( - ROW_DIFF_ABSOLUTE_TOLERANCE_KEY, - ROW_DIFF_RELATIVE_TOLERANCE_KEY, - ROW_DIFF_TOLERANCE_KEYS, -) -from sqlbuild.executor.diff.models import RowDiffSamplingOverride - - -def resolve_row_diff_sampling( - *, - raw_row_limit: object, - raw_seed: object, - override: RowDiffSamplingOverride, - label: str, -) -> RowDiffSampling | None: - """Resolve model sampling configuration with invocation overrides.""" - - if override.exhaustive: - return None - row_limit: int | None = _parse_optional_non_negative_integer( - raw=override.row_limit if override.row_limit is not None else raw_row_limit, - label=f"{label}.row_diff_sample_rows", - ) - if row_limit in (None, 0): - return None - seed: int | None = _parse_optional_integer( - raw=override.seed if override.seed is not None else raw_seed, - label=f"{label}.row_diff_sample_seed", - ) - return RowDiffSampling(row_limit=row_limit, seed=seed or 0) - - -def parse_row_diff_tolerances( - *, - raw: object, - label: str = "row_diff_tolerances", -) -> RowDiffTolerances: - """Parse raw model config into typed row diff tolerances.""" - - if raw is None: - return RowDiffTolerances() - if not isinstance(raw, dict): - raise ExecutorInputError(f"{label} must be a mapping", code="X401") - raw_mapping: dict[str, object] = cast(dict[str, object], raw) - - by_type: dict[str, RowDiffTolerance] = _parse_tolerance_section( - raw=raw_mapping.get("by_type"), - label=f"{label}.by_type", - normalize_key=True, - ) - by_column: dict[str, RowDiffTolerance] = _parse_tolerance_section( - raw=raw_mapping.get("by_column"), - label=f"{label}.by_column", - normalize_key=False, - ) - return RowDiffTolerances(by_type=by_type, by_column=by_column) - - -def parse_cli_tolerance_overrides(*, values: tuple[str, ...]) -> RowDiffTolerances: - """Parse repeated COLUMN:KIND=VALUE CLI tolerance specifications.""" - - raw_by_column: dict[str, dict[str, str]] = {} - for value in values: - column_and_kind, separator, amount = value.partition("=") - column, kind_separator, kind = column_and_kind.partition(":") - if ( - not separator - or not kind_separator - or not column - or kind not in ROW_DIFF_TOLERANCE_KEYS - or not amount - ): - raise ExecutorInputError( - "diff --tolerance must use COLUMN:absolute=VALUE or COLUMN:relative=VALUE", - code="X408", - ) - raw_by_column.setdefault(column, {})[kind] = amount - return parse_row_diff_tolerances(raw={"by_column": raw_by_column}) - - -def merge_row_diff_tolerances( - *, base: RowDiffTolerances, override: RowDiffTolerances | None -) -> RowDiffTolerances: - """Overlay invocation tolerance rules on compiled model rules.""" - - if override is None: - return base - return RowDiffTolerances( - by_type={**base.by_type, **override.by_type}, - by_column={**base.by_column, **override.by_column}, - sampling=base.sampling, - ) - - -def _parse_tolerance_section( - *, - raw: object, - label: str, - normalize_key: bool, -) -> dict[str, RowDiffTolerance]: - if raw is None: - return {} - if not isinstance(raw, dict): - raise ExecutorInputError(f"{label} must be a mapping", code="X401") - raw_mapping: dict[object, object] = cast(dict[object, object], raw) - - parsed: dict[str, RowDiffTolerance] = {} - key: object - rule: object - for key, rule in raw_mapping.items(): - if not isinstance(key, str) or not key: - raise ExecutorInputError(f"{label} keys must be non-empty strings", code="X402") - parsed_key: str = key.lower() if normalize_key else key - parsed[parsed_key] = _parse_tolerance_rule(raw=rule, label=f"{label}.{key}") - return parsed - - -def _parse_tolerance_rule(*, raw: object, label: str) -> RowDiffTolerance: - if not isinstance(raw, dict): - raise ExecutorInputError(f"{label} must be a mapping", code="X401") - raw_mapping: dict[str, object] = cast(dict[str, object], raw) - - unsupported_keys: tuple[str, ...] = tuple( - str(key) for key in raw_mapping if key not in ROW_DIFF_TOLERANCE_KEYS - ) - if unsupported_keys: - unsupported: str = ", ".join(unsupported_keys) - raise ExecutorInputError(f"{label} contains unsupported keys: {unsupported}", code="X403") - - absolute: Decimal | None = _parse_optional_decimal( - raw=raw_mapping.get(ROW_DIFF_ABSOLUTE_TOLERANCE_KEY), - label=f"{label}.absolute", - ) - relative: Decimal | None = _parse_optional_decimal( - raw=raw_mapping.get(ROW_DIFF_RELATIVE_TOLERANCE_KEY), - label=f"{label}.relative", - ) - if absolute is None and relative is None: - raise ExecutorInputError(f"{label} must define absolute or relative", code="X404") - return RowDiffTolerance(absolute=absolute, relative=relative) - - -def _parse_optional_decimal(*, raw: object, label: str) -> Decimal | None: - if raw is None: - return None - if isinstance(raw, bool): - raise ExecutorInputError(f"{label} must be numeric", code="X405") - if not isinstance(raw, str | int | float | Decimal): - raise ExecutorInputError(f"{label} must be numeric", code="X405") - try: - return Decimal(str(raw)) - except InvalidOperation as error: - raise ExecutorInputError(f"{label} must be numeric", code="X405") from error - - -def _parse_optional_integer(*, raw: object, label: str) -> int | None: - if raw is None: - return None - if isinstance(raw, bool) or not isinstance(raw, int): - raise ExecutorInputError(f"{label} must be an integer", code="X406") - return raw - - -def _parse_optional_non_negative_integer(*, raw: object, label: str) -> int | None: - value: int | None = _parse_optional_integer(raw=raw, label=label) - if value is not None and value < 0: - raise ExecutorInputError(f"{label} must be zero or greater", code="X407") - return value diff --git a/src/sqlbuild/executor/diff/_helpers/execution.py b/src/sqlbuild/executor/diff/_helpers/execution.py index 4ed4af7c97..72d630ed94 100644 --- a/src/sqlbuild/executor/diff/_helpers/execution.py +++ b/src/sqlbuild/executor/diff/_helpers/execution.py @@ -5,6 +5,11 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import ( + merge_row_diff_tolerances, + parse_row_diff_tolerances, + resolve_row_diff_sampling, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ( ColumnInfo, @@ -16,11 +21,6 @@ ) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.diff._helpers.bounds import resolve_bounded_cursors -from sqlbuild.executor.diff._helpers.config import ( - merge_row_diff_tolerances, - parse_row_diff_tolerances, - resolve_row_diff_sampling, -) from sqlbuild.executor.diff._helpers.selection import ( get_row_diff_exclude_columns, get_unique_key, diff --git a/src/sqlbuild/executor/diff/constants.py b/src/sqlbuild/executor/diff/constants.py index 5387092604..807a3ff99f 100644 --- a/src/sqlbuild/executor/diff/constants.py +++ b/src/sqlbuild/executor/diff/constants.py @@ -1,9 +1,4 @@ """Constants for diff execution.""" -ROW_DIFF_ABSOLUTE_TOLERANCE_KEY: str = "absolute" -ROW_DIFF_RELATIVE_TOLERANCE_KEY: str = "relative" DIFF_INPUT_KIND_MODEL: str = "model" DIFF_INPUT_KIND_QUERY: str = "query" -ROW_DIFF_TOLERANCE_KEYS: frozenset[str] = frozenset( - (ROW_DIFF_ABSOLUTE_TOLERANCE_KEY, ROW_DIFF_RELATIVE_TOLERANCE_KEY) -) diff --git a/src/sqlbuild/executor/diff/main/_config.py b/src/sqlbuild/executor/diff/main/_config.py deleted file mode 100644 index 6177f157ad..0000000000 --- a/src/sqlbuild/executor/diff/main/_config.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public diff invocation configuration entrypoints.""" - -from sqlbuild.adapter.contract.models import RowDiffTolerances -from sqlbuild.executor.diff._helpers.config import ( - parse_cli_tolerance_overrides as _parse_cli_tolerance_overrides, -) - - -def parse_cli_tolerance_overrides(*, values: tuple[str, ...]) -> RowDiffTolerances: - """Parse repeated invocation-level per-column tolerance rules.""" - - return _parse_cli_tolerance_overrides(values=values) diff --git a/tests/unit/src/sqlbuild/executor/diff/test_config.py b/tests/unit/src/sqlbuild/executor/diff/test_config.py index 2880938ca0..5634955258 100644 --- a/tests/unit/src/sqlbuild/executor/diff/test_config.py +++ b/tests/unit/src/sqlbuild/executor/diff/test_config.py @@ -4,13 +4,13 @@ import pytest -from sqlbuild.adapter.contract.models import RowDiffSampling, RowDiffTolerance, RowDiffTolerances -from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.executor.diff._helpers.config import ( +from sqlbuild._native import ( parse_cli_tolerance_overrides, parse_row_diff_tolerances, resolve_row_diff_sampling, ) +from sqlbuild.adapter.contract.models import RowDiffSampling, RowDiffTolerance, RowDiffTolerances +from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.diff.models import RowDiffSamplingOverride from tests.unit.src.sqlbuild.executor.diff._test_types import ( CliToleranceTestCase, From c61b9326e7e5ee8b85e2ee31eed5182c02221a99 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:51:14 +0100 Subject: [PATCH 04/11] refactor: aggregate build result counts natively CLI result documents no longer import executor Python to recount a partial build; the native aggregation is shared with the executor. --- .../src/build_aggregate/constants.rs | 10 ++ .../main/aggregate_build_counts.rs | 73 ++++++++++ .../src/build_aggregate/main/mod.rs | 1 + .../sqlbuild-core/src/build_aggregate/mod.rs | 5 + .../src/build_aggregate/models.rs | 34 +++++ crates/sqlbuild-core/src/lib.rs | 1 + .../_helpers/result_output/aggregation.rs | 120 ++++++++++++++++ .../_helpers/result_output/documents.rs | 16 +-- .../bindings/_helpers/result_output/mod.rs | 1 + .../_helpers/result_output/registration.rs | 3 +- src/sqlbuild/_native.pyi | 12 ++ .../executor/build/_helpers/aggregation.py | 19 --- .../executor/build/main/_aggregate_result.py | 133 ------------------ src/sqlbuild/executor/build/main/_execute.py | 2 +- 14 files changed, 263 insertions(+), 167 deletions(-) create mode 100644 crates/sqlbuild-core/src/build_aggregate/constants.rs create mode 100644 crates/sqlbuild-core/src/build_aggregate/main/aggregate_build_counts.rs create mode 100644 crates/sqlbuild-core/src/build_aggregate/main/mod.rs create mode 100644 crates/sqlbuild-core/src/build_aggregate/mod.rs create mode 100644 crates/sqlbuild-core/src/build_aggregate/models.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs delete mode 100644 src/sqlbuild/executor/build/_helpers/aggregation.py delete mode 100644 src/sqlbuild/executor/build/main/_aggregate_result.py diff --git a/crates/sqlbuild-core/src/build_aggregate/constants.rs b/crates/sqlbuild-core/src/build_aggregate/constants.rs new file mode 100644 index 0000000000..0ac789c3c3 --- /dev/null +++ b/crates/sqlbuild-core/src/build_aggregate/constants.rs @@ -0,0 +1,10 @@ +//! Result status and outcome values the aggregation reads. + +pub(crate) const SUCCESS: &str = "success"; +pub(crate) const FAILED: &str = "failed"; +pub(crate) const SKIPPED: &str = "skipped"; +pub(crate) const TEST_PASS: &str = "pass"; +pub(crate) const AUDIT_WARN: &str = "warn"; +pub(crate) const AUDIT_ERROR: &str = "error"; +pub(crate) const AUDIT_INSUFFICIENT: &str = "insufficient"; +pub(crate) const SOURCE_FRESHNESS_BLOCKED_CODE: &str = "B005"; diff --git a/crates/sqlbuild-core/src/build_aggregate/main/aggregate_build_counts.rs b/crates/sqlbuild-core/src/build_aggregate/main/aggregate_build_counts.rs new file mode 100644 index 0000000000..f749904e5b --- /dev/null +++ b/crates/sqlbuild-core/src/build_aggregate/main/aggregate_build_counts.rs @@ -0,0 +1,73 @@ +//! Aggregate counts and overall status of one build. + +use crate::build_aggregate::constants::{ + AUDIT_ERROR, AUDIT_INSUFFICIENT, AUDIT_WARN, FAILED, SKIPPED, SOURCE_FRESHNESS_BLOCKED_CODE, + SUCCESS, TEST_PASS, +}; +use crate::build_aggregate::models::{BuildCounts, BuildResultFacts}; + +/// Success, failure, skipped, warning and insufficient counts, and whether the build failed. +#[must_use] +pub fn aggregate_build_counts(facts: &BuildResultFacts) -> BuildCounts { + let mut counts: BuildCounts = BuildCounts::default(); + for model in &facts.models { + match model.status.as_str() { + SUCCESS => counts.success += 1, + FAILED => counts.failure += 1, + SKIPPED => { + counts.skipped += 1; + if model.error_code.as_deref() == Some(SOURCE_FRESHNESS_BLOCKED_CODE) { + counts.failure += 1; + } + } + _ => {} + } + counts.warning += model.warning_count + matching(&model.audit_outcomes, AUDIT_WARN); + } + let statuses = facts + .seed_statuses + .iter() + .chain(facts.functions.iter().map(|(status, _)| status)) + .chain(&facts.load_statuses); + for status in statuses { + counts.success += usize::from(status == SUCCESS); + counts.failure += usize::from(status == FAILED); + counts.skipped += usize::from(status == SKIPPED); + } + counts.warning += facts + .functions + .iter() + .map(|(_, warnings)| warnings) + .sum::(); + for outcome in &facts.test_outcomes { + if outcome == TEST_PASS { + counts.success += 1; + } else { + counts.failure += 1; + } + } + let model_audits = facts.models.iter().flat_map(|model| &model.audit_outcomes); + counts.insufficient = facts + .source_audit_outcomes + .iter() + .chain(&facts.end_audit_outcomes) + .chain(model_audits) + .filter(|outcome| *outcome == AUDIT_INSUFFICIENT) + .count(); + counts.failure += matching(&facts.source_audit_outcomes, AUDIT_ERROR); + counts.warning += matching(&facts.source_audit_outcomes, AUDIT_WARN) + + matching(&facts.end_audit_outcomes, AUDIT_WARN); + counts.failed = counts.failure > 0 + || facts + .end_audit_outcomes + .iter() + .any(|outcome| outcome == AUDIT_ERROR); + counts +} + +fn matching(outcomes: &[String], expected: &str) -> usize { + outcomes + .iter() + .filter(|outcome| *outcome == expected) + .count() +} diff --git a/crates/sqlbuild-core/src/build_aggregate/main/mod.rs b/crates/sqlbuild-core/src/build_aggregate/main/mod.rs new file mode 100644 index 0000000000..e94806f546 --- /dev/null +++ b/crates/sqlbuild-core/src/build_aggregate/main/mod.rs @@ -0,0 +1 @@ +pub mod aggregate_build_counts; diff --git a/crates/sqlbuild-core/src/build_aggregate/mod.rs b/crates/sqlbuild-core/src/build_aggregate/mod.rs new file mode 100644 index 0000000000..4bd1773ec1 --- /dev/null +++ b/crates/sqlbuild-core/src/build_aggregate/mod.rs @@ -0,0 +1,5 @@ +//! Aggregate counts and overall status of one build's execution results. + +mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-core/src/build_aggregate/models.rs b/crates/sqlbuild-core/src/build_aggregate/models.rs new file mode 100644 index 0000000000..2ceed85105 --- /dev/null +++ b/crates/sqlbuild-core/src/build_aggregate/models.rs @@ -0,0 +1,34 @@ +//! Build result facts and their aggregate counts. + +/// The status, error code, warning count and audit outcomes of one model result. +#[derive(Debug, Clone, Default)] +pub struct ModelResultFacts { + pub status: String, + pub error_code: Option, + pub warning_count: usize, + pub audit_outcomes: Vec, +} + +/// The statuses and outcomes of every result one build produced. +#[derive(Debug, Clone, Default)] +pub struct BuildResultFacts { + pub models: Vec, + pub seed_statuses: Vec, + /// `(status, warning count)` of each function result. + pub functions: Vec<(String, usize)>, + pub load_statuses: Vec, + pub test_outcomes: Vec, + pub source_audit_outcomes: Vec, + pub end_audit_outcomes: Vec, +} + +/// Aggregate counts and whether the build failed. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct BuildCounts { + pub success: usize, + pub failure: usize, + pub skipped: usize, + pub warning: usize, + pub insufficient: usize, + pub failed: bool, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 2e9cf4e654..f2911bf659 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -3,6 +3,7 @@ #![forbid(unsafe_code)] pub mod audit_command; +pub mod build_aggregate; pub mod build_command; pub mod build_progress; pub mod check_command; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs new file mode 100644 index 0000000000..9c892671f1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs @@ -0,0 +1,120 @@ +//! Build `BuildExecutionResult` with aggregate counts and overall status from typed results. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_core::build_aggregate::main::aggregate_build_counts::aggregate_build_counts; +use sqlbuild_core::build_aggregate::models::{BuildCounts, BuildResultFacts, ModelResultFacts}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const BUILD_MODELS_MODULE: &str = "sqlbuild.executor.build.models"; +const BUILD_TYPES_MODULE: &str = "sqlbuild.executor.build.types"; +const RESULT_FIELDS: [&str; 7] = [ + "model_results", + "seed_results", + "function_results", + "load_results", + "test_results", + "source_audit_results", + "end_audit_results", +]; + +fn texts(results: &Bound<'_, PyAny>, field: &str) -> PyResult> { + results + .try_iter()? + .map(|result| result?.getattr(field)?.str()?.extract()) + .collect() +} + +fn facts(sections: &[Bound<'_, PyAny>]) -> PyResult { + let models: Vec = sections[0] + .try_iter()? + .map(|model| { + let model: Bound<'_, PyAny> = model?; + Ok(ModelResultFacts { + status: model.getattr("status")?.str()?.extract()?, + error_code: model.getattr("error_code")?.extract()?, + warning_count: model.getattr("warning_messages")?.len()?, + audit_outcomes: texts(&model.getattr("audit_results")?, "outcome")?, + }) + }) + .collect::>()?; + let functions: Vec<(String, usize)> = sections[2] + .try_iter()? + .map(|function| { + let function: Bound<'_, PyAny> = function?; + Ok(( + function.getattr("status")?.str()?.extract()?, + function.getattr("warning_messages")?.len()?, + )) + }) + .collect::>()?; + Ok(BuildResultFacts { + models, + seed_statuses: texts(§ions[1], "status")?, + functions, + load_statuses: texts(§ions[3], "status")?, + test_outcomes: texts(§ions[4], "outcome")?, + source_audit_outcomes: texts(§ions[5], "outcome")?, + end_audit_outcomes: texts(§ions[6], "outcome")?, + }) +} + +/// `BuildExecutionResult` over the seven result tuples, in `RESULT_FIELDS` order. +pub(crate) fn aggregated_build_result<'py>( + py: Python<'py>, + sections: &[Bound<'py, PyAny>], +) -> PyResult> { + let counts: BuildCounts = aggregate_build_counts(&facts(sections)?); + let status: Bound<'py, PyAny> = imported(py, BUILD_TYPES_MODULE, "BuildStatus")? + .getattr(if counts.failed { "FAILED" } else { "SUCCESS" })?; + let mut keywords: Vec<(&str, Bound<'py, PyAny>)> = vec![("status", status)]; + keywords.extend(RESULT_FIELDS.iter().copied().zip(sections.iter().cloned())); + keywords.extend([ + ("success_count", counts.success.into_bound_py_any(py)?), + ("failure_count", counts.failure.into_bound_py_any(py)?), + ("skipped_count", counts.skipped.into_bound_py_any(py)?), + ("warning_count", counts.warning.into_bound_py_any(py)?), + ( + "insufficient_count", + counts.insufficient.into_bound_py_any(py)?, + ), + ]); + call_with( + &imported(py, BUILD_MODELS_MODULE, "BuildExecutionResult")?, + &keywords, + ) +} + +/// Compute aggregate counts and overall build status. +#[pyfunction] +#[pyo3(signature = (**options))] +fn aggregate_build_result<'py>( + options: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("aggregate_build_result", options)?; + let sections: Vec> = RESULT_FIELDS + .iter() + .map(|field| request.get(field)) + .collect::>()?; + aggregated_build_result(request.py(), §ions) +} + +/// `aggregate_build_result` over retained result lists, as tuples. +pub(crate) fn aggregated_from_lists<'py>( + py: Python<'py>, + retained: &[Vec>], +) -> PyResult> { + let sections: Vec> = retained + .iter() + .map(|kept| Ok(PyTuple::new(py, kept)?.into_any())) + .collect::>()?; + aggregated_build_result(py, §ions) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(aggregate_build_result, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs index 59b16c0f77..4b5798fdde 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs @@ -4,11 +4,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; use crate::bindings::_helpers::compile_outputs::json_report::indented_json; use crate::bindings::_helpers::plan_output::values::{any, dict, items, text}; +use crate::bindings::_helpers::result_output::aggregation::aggregated_from_lists; use crate::bindings::_helpers::result_output::assets::{ clone_asset, function_assets, load_assets, model_assets, python_node_assets, seed_assets, without_none, @@ -19,7 +18,6 @@ use crate::bindings::_helpers::result_output::checks::{ use crate::bindings::_helpers::result_output::projection::{Projection, rounded}; const JSON_VERSION: i64 = 1; -const AGGREGATE_MODULE: &str = "sqlbuild.executor.build.main._aggregate_result"; const FAILED: &str = "failed"; const SUCCESS: &str = "success"; const SKIPPED: &str = "skipped"; @@ -332,15 +330,7 @@ fn build_json<'py>( assets.extend(load_assets(&projection, &retained[3])?); assets.extend(python_node_assets(&projection, &nodes)?); let counted: Bound<'py, PyAny> = if partial { - let keywords: Vec<(&str, Bound<'py, PyAny>)> = BUILD_RESULT_FIELDS - .iter() - .zip(&retained) - .map(|(field, kept)| Ok((*field, PyTuple::new(py, kept)?.into_any()))) - .collect::>()?; - call_with( - &imported(py, AGGREGATE_MODULE, "aggregate_build_result")?, - &keywords, - )? + aggregated_from_lists(py, &retained)? } else { result.clone() }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs index 410a105496..ac7be681a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod aggregation; pub(crate) mod assets; pub(crate) mod checks; pub(crate) mod documents; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs index 5392910355..b94c4fbf40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs @@ -3,12 +3,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::result_output::{ - documents, event_writer, integration_envelope, resource_ids, terminal_index, + aggregation, documents, event_writer, integration_envelope, resource_ids, terminal_index, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { terminal_index::register(module)?; resource_ids::register(module)?; + aggregation::register(module)?; integration_envelope::register(module)?; event_writer::register(module)?; documents::register(module) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 1540a6f7c5..619e0e7b20 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -295,6 +295,7 @@ from sqlbuild.diagnostics.models import ProcessResourceUsage from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import ( BuildExecutionResult, + FunctionExecutionResult, SchedulerState, SeedExecutionResult, ) @@ -306,6 +307,7 @@ from sqlbuild.executor.python_nodes.models import ( PythonCheckExecutionResult, PythonNodeExecutionResult, ) +from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scenario.models import ScenarioRunResult, ScenarioSnapshotCaptureRunResult from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner @@ -4927,6 +4929,16 @@ def resolve_row_diff_sampling( def merge_row_diff_tolerances( *, base: RowDiffTolerances, override: RowDiffTolerances | None ) -> RowDiffTolerances: ... +def aggregate_build_result( + *, + model_results: tuple[ModelExecutionResult, ...], + seed_results: tuple[SeedExecutionResult, ...], + function_results: tuple[FunctionExecutionResult, ...], + load_results: tuple[LoadExecutionResult, ...], + test_results: tuple[SqlTestExecutionResult, ...], + source_audit_results: tuple[AuditExecutionResult, ...], + end_audit_results: tuple[AuditExecutionResult, ...], +) -> BuildExecutionResult: ... def fit_artifact_logical_name( *, logical_name: str, fixed_prefix: str, identifier_limit: int, artifact_label: str ) -> str: ... diff --git a/src/sqlbuild/executor/build/_helpers/aggregation.py b/src/sqlbuild/executor/build/_helpers/aggregation.py deleted file mode 100644 index f76e65b1af..0000000000 --- a/src/sqlbuild/executor/build/_helpers/aggregation.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Build result aggregation helpers.""" - -from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.executor.run.models import ModelExecutionResult - - -def count_insufficient_audits( - *, - model_results: tuple[ModelExecutionResult, ...], - source_audit_results: tuple[AuditExecutionResult, ...], - end_audit_results: tuple[AuditExecutionResult, ...], -) -> int: - """Count visible non-blocking insufficient measurement outcomes.""" - - results: list[AuditExecutionResult] = [*source_audit_results, *end_audit_results] - for model_result in model_results: - results.extend(model_result.audit_results) - return sum(result.outcome == AuditOutcome.INSUFFICIENT for result in results) diff --git a/src/sqlbuild/executor/build/main/_aggregate_result.py b/src/sqlbuild/executor/build/main/_aggregate_result.py deleted file mode 100644 index 861cdbd9fd..0000000000 --- a/src/sqlbuild/executor/build/main/_aggregate_result.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Public entry for typed build result aggregation.""" - -from __future__ import annotations - -from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.executor.build._helpers.aggregation import count_insufficient_audits -from sqlbuild.executor.build.constants import BUILD_SOURCE_FRESHNESS_BLOCKED_CODE -from sqlbuild.executor.build.models import ( - BuildExecutionResult, - FunctionExecutionResult, - SeedExecutionResult, -) -from sqlbuild.executor.build.types import BuildStatus -from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.run.models import ModelExecutionResult -from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.executor.testing.models import SqlTestExecutionResult -from sqlbuild.executor.testing.types import SqlTestOutcome - - -def aggregate_build_result( - *, - model_results: tuple[ModelExecutionResult, ...], - seed_results: tuple[SeedExecutionResult, ...], - function_results: tuple[FunctionExecutionResult, ...], - load_results: tuple[LoadExecutionResult, ...], - test_results: tuple[SqlTestExecutionResult, ...], - source_audit_results: tuple[AuditExecutionResult, ...], - end_audit_results: tuple[AuditExecutionResult, ...], -) -> BuildExecutionResult: - """Compute aggregate counts and overall build status.""" - - resource_counts: tuple[int, int, int, int] = _aggregate_resources( - model_results=model_results, - seed_results=seed_results, - function_results=function_results, - load_results=load_results, - test_results=test_results, - ) - success_count: int = resource_counts[0] - failure_count: int = resource_counts[1] - skipped_count: int = resource_counts[2] - warning_count: int = resource_counts[3] - insufficient_count: int = count_insufficient_audits( - model_results=model_results, - source_audit_results=source_audit_results, - end_audit_results=end_audit_results, - ) - audit_counts: tuple[int, int, bool] = _aggregate_audits( - source_audit_results=source_audit_results, - end_audit_results=end_audit_results, - ) - failure_count += audit_counts[0] - warning_count += audit_counts[1] - status: BuildStatus = ( - BuildStatus.FAILED if failure_count > 0 or audit_counts[2] else BuildStatus.SUCCESS - ) - return BuildExecutionResult( - status=status, - model_results=model_results, - seed_results=seed_results, - function_results=function_results, - load_results=load_results, - test_results=test_results, - source_audit_results=source_audit_results, - end_audit_results=end_audit_results, - success_count=success_count, - failure_count=failure_count, - skipped_count=skipped_count, - warning_count=warning_count, - insufficient_count=insufficient_count, - ) - - -def _aggregate_resources( - *, - model_results: tuple[ModelExecutionResult, ...], - seed_results: tuple[SeedExecutionResult, ...], - function_results: tuple[FunctionExecutionResult, ...], - load_results: tuple[LoadExecutionResult, ...], - test_results: tuple[SqlTestExecutionResult, ...], -) -> tuple[int, int, int, int]: - success_count: int = 0 - failure_count: int = 0 - skipped_count: int = 0 - warning_count: int = 0 - for model_result in model_results: - if model_result.status == ExecutionStatus.SUCCESS: - success_count += 1 - elif model_result.status == ExecutionStatus.FAILED: - failure_count += 1 - elif model_result.status == ExecutionStatus.SKIPPED: - skipped_count += 1 - if model_result.error_code == BUILD_SOURCE_FRESHNESS_BLOCKED_CODE: - failure_count += 1 - warning_count += len(model_result.warning_messages) - warning_count += sum( - audit_result.outcome == AuditOutcome.WARN for audit_result in model_result.audit_results - ) - for seed_result in seed_results: - success_count += seed_result.status == ExecutionStatus.SUCCESS - failure_count += seed_result.status == ExecutionStatus.FAILED - skipped_count += seed_result.status == ExecutionStatus.SKIPPED - for function_result in function_results: - success_count += function_result.status == ExecutionStatus.SUCCESS - failure_count += function_result.status == ExecutionStatus.FAILED - skipped_count += function_result.status == ExecutionStatus.SKIPPED - warning_count += len(function_result.warning_messages) - for load_result in load_results: - success_count += load_result.status == ExecutionStatus.SUCCESS - failure_count += load_result.status == ExecutionStatus.FAILED - skipped_count += load_result.status == ExecutionStatus.SKIPPED - for test_result in test_results: - success_count += test_result.outcome == SqlTestOutcome.PASS - failure_count += test_result.outcome != SqlTestOutcome.PASS - return success_count, failure_count, skipped_count, warning_count - - -def _aggregate_audits( - *, - source_audit_results: tuple[AuditExecutionResult, ...], - end_audit_results: tuple[AuditExecutionResult, ...], -) -> tuple[int, int, bool]: - source_failures: int = sum( - result.outcome == AuditOutcome.ERROR for result in source_audit_results - ) - warnings: int = sum( - result.outcome == AuditOutcome.WARN - for result in (*source_audit_results, *end_audit_results) - ) - any_end_error: bool = any(result.outcome == AuditOutcome.ERROR for result in end_audit_results) - return source_failures, warnings, any_end_error diff --git a/src/sqlbuild/executor/build/main/_execute.py b/src/sqlbuild/executor/build/main/_execute.py index ba81c28583..27b9768f1b 100644 --- a/src/sqlbuild/executor/build/main/_execute.py +++ b/src/sqlbuild/executor/build/main/_execute.py @@ -5,6 +5,7 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import aggregate_build_result from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.relation_metadata_cache_scope import ( relation_metadata_cache_scope, @@ -24,7 +25,6 @@ reconcile_retention_after_build, ) from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler -from sqlbuild.executor.build.main._aggregate_result import aggregate_build_result from sqlbuild.executor.build.models import ( BuildCallbacks, BuildCustomizations, From 61f59b3a25ecb0907939ed57725d83b4c8463aa8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:00:21 +0100 Subject: [PATCH 05/11] refactor: resolve Dagster selections and load DAG artifacts natively The Dagster integration resolves SQLBuild selectors over the DAG artifact with the native selector parser and graph closures instead of a Python re-implementation, and Dagster and Rivers share one native DAG artifact loader in place of two duplicated Python copies. --- .../src/dag_artifact/_helpers/mod.rs | 1 + .../src/dag_artifact/_helpers/selection.rs | 219 ++++++++++++++++++ .../src/dag_artifact/constants.rs | 7 + .../src/dag_artifact/errors.rs | 11 + .../src/dag_artifact/main/mod.rs | 1 + .../main/resolve_dag_selection.rs | 50 ++++ .../src/dag_artifact/models.rs | 35 +++ .../src/bindings/_helpers/dag_artifact/mod.rs | 1 + .../_helpers/dag_artifact/registration.rs | 5 +- .../_helpers/dag_artifact/selection.rs | 218 +++++++++++++++++ src/sqlbuild/_native.pyi | 6 + .../integrations/dagster/_helpers/dag.py | 44 ---- .../dagster/_helpers/selection.py | 197 +--------------- .../dagster/classes/sqlbuild_cli_resource.py | 5 +- .../dagster/main/_sqlbuild_assets.py | 4 +- .../dagster/main/_sqlbuild_scenario_checks.py | 4 +- .../integrations/rivers/_helpers/dag.py | 44 ---- .../rivers/main/_sqlbuild_assets.py | 4 +- .../rivers/test_sqlbuild_assets.py | 6 +- 19 files changed, 567 insertions(+), 295 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/_helpers/selection.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/main/resolve_dag_selection.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs delete mode 100644 src/sqlbuild/integrations/dagster/_helpers/dag.py delete mode 100644 src/sqlbuild/integrations/rivers/_helpers/dag.py diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs index 7dec7aa968..8d9c11ec79 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod checks; pub(crate) mod edges; pub(crate) mod nodes; +pub(crate) mod selection; pub(crate) mod values; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/selection.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/selection.rs new file mode 100644 index 0000000000..8148ad5255 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/selection.rs @@ -0,0 +1,219 @@ +//! Breadth-first closures and directed path nodes over string-keyed DAG edges. + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +use sqlbuild_core::text::main::python_isspace::python_isspace; +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +use crate::dag_artifact::constants::{ + SELECTOR_KIND_CHECK, SELECTOR_KIND_NAME, SELECTOR_KIND_PATH, SELECTOR_KIND_SEPARATOR, + SELECTOR_KIND_TAG, SELECTOR_KIND_TEST, SELECTOR_MODELS_PREFIX, +}; +use crate::dag_artifact::errors::DagSelectionError; +use crate::dag_artifact::models::{DagSelectionIndex, DagSelectionNode}; +use crate::graph::main::parse_selector::parse_selector; +use crate::graph::models::ParsedSelector; + +/// Upstream and downstream neighbours of each node id. +pub(crate) type Adjacency = BTreeMap>; + +/// Every id reachable from `starts`, plus the starts when `include_starts`. +pub(crate) fn closure( + starts: &BTreeSet, + edges: &Adjacency, + include_starts: bool, +) -> BTreeSet { + let mut visited: BTreeSet = if include_starts { + starts.clone() + } else { + BTreeSet::new() + }; + let mut frontier: VecDeque<&String> = starts.iter().collect(); + while let Some(current) = frontier.pop_front() { + for neighbor in edges.get(current).into_iter().flatten() { + if visited.insert(neighbor.clone()) { + frontier.push_back(neighbor); + } + } + } + visited +} + +/// Ids on directed paths from `start` to `end`, or `None` when `end` is not downstream. +pub(crate) fn path_nodes( + (start, end): (&str, &str), + (upstream, downstream): (&Adjacency, &Adjacency), +) -> Option> { + let reachable: BTreeSet = + closure(&BTreeSet::from([start.to_owned()]), downstream, false); + if !reachable.contains(end) { + return None; + } + let mut upstream_of_end: BTreeSet = BTreeSet::new(); + let mut stack: Vec = vec![end.to_owned()]; + while let Some(current) = stack.pop() { + if !upstream_of_end.insert(current.clone()) { + continue; + } + for parent in upstream.get(¤t).into_iter().flatten() { + if reachable.contains(parent) || parent == start { + stack.push(parent.clone()); + } + } + } + let mut nodes: BTreeSet = reachable.intersection(&upstream_of_end).cloned().collect(); + nodes.insert(start.to_owned()); + nodes.insert(end.to_owned()); + Some(nodes) +} + +pub(crate) fn resolve_expression( + index: &DagSelectionIndex<'_>, + expression: &str, +) -> Result, DagSelectionError> { + let mut selected: BTreeSet = BTreeSet::new(); + for token in expression + .split(python_isspace) + .filter(|token| !token.is_empty()) + { + let mut token_selection: Option> = None; + for part in token.split(',') { + let matched: BTreeSet = resolve_atomic(index, part)?; + token_selection = Some(match token_selection { + None => matched, + Some(previous) => previous.intersection(&matched).cloned().collect(), + }); + } + selected.extend(token_selection.unwrap_or_default()); + } + Ok(selected) +} + +fn resolve_atomic( + index: &DagSelectionIndex<'_>, + raw: &str, +) -> Result, DagSelectionError> { + match parse_selector(raw).map_err(DagSelectionError::Selector)? { + ParsedSelector::Path { + start, + end, + upstream, + downstream, + } => { + let start_id: String = node_id_for_name(index, &start)?; + let end_id: String = node_id_for_name(index, &end)?; + let mut matched: BTreeSet = + path_nodes((&start_id, &end_id), (&index.upstream, &index.downstream)).ok_or_else( + || { + DagSelectionError::Input(format!( + "no SQLBuild DAG path exists between {} and {}", + python_str_repr(&start_id), + python_str_repr(&end_id) + )) + }, + )?; + if upstream { + matched.extend(closure(&BTreeSet::from([start_id]), &index.upstream, true)); + } + if downstream { + matched.extend(closure(&BTreeSet::from([end_id]), &index.downstream, true)); + } + Ok(matched) + } + ParsedSelector::Kind { + kind, + value, + upstream, + downstream, + } => { + let mut matched: BTreeSet = match_kind(index, &kind, &value)?; + let anchors: BTreeSet = matched.clone(); + if upstream { + matched.extend(closure(&anchors, &index.upstream, true)); + } + if downstream { + matched.extend(closure(&anchors, &index.downstream, true)); + } + Ok(matched) + } + } +} + +fn match_kind( + index: &DagSelectionIndex<'_>, + kind: &str, + value: &str, +) -> Result, DagSelectionError> { + let ids = |matches: &dyn Fn(&DagSelectionNode) -> bool| -> BTreeSet { + index + .nodes + .iter() + .filter(|(_, node)| matches(node)) + .map(|(id, _)| (*id).to_owned()) + .collect() + }; + Ok(match kind { + SELECTOR_KIND_TEST => { + return Err(DagSelectionError::UnitTest(format!( + "{SELECTOR_KIND_TEST}{SELECTOR_KIND_SEPARATOR}{value}" + ))); + } + SELECTOR_KIND_TAG => ids(&|node| node.tags.iter().any(|tag| tag == value)), + SELECTOR_KIND_PATH => { + let folder: String = value.replace('\\', "/"); + let folder: &str = folder.trim_matches('/'); + ids(&|node| path_is_under(node.path.as_deref(), folder)) + } + SELECTOR_KIND_CHECK => index + .graph + .checks + .iter() + .filter(|check| check.name == value || check.id == value) + .flat_map(|check| check.checked_asset_ids.iter().cloned()) + .collect(), + SELECTOR_KIND_NAME => ids(&|node| node.name == value), + _ => ids(&|node| node.name == value && node.kind == kind), + }) +} + +fn path_is_under(path: Option<&str>, folder: &str) -> bool { + let Some(path) = path else { + return false; + }; + let normalized: String = path.replace('\\', "/"); + let normalized: &str = normalized.trim_matches('/'); + let prefixed: String = if !folder.is_empty() && !folder.starts_with(SELECTOR_MODELS_PREFIX) { + format!("{SELECTOR_MODELS_PREFIX}{folder}") + } else { + folder.to_owned() + }; + [folder, prefixed.as_str()] + .into_iter() + .filter(|candidate| !candidate.is_empty()) + .any(|candidate| { + normalized == candidate + || normalized + .strip_prefix(candidate) + .is_some_and(|rest| rest.starts_with('/')) + }) +} + +fn node_id_for_name( + index: &DagSelectionIndex<'_>, + name: &str, +) -> Result { + let matching: Vec<&str> = index + .nodes + .iter() + .filter(|(_, node)| node.name == name) + .map(|(id, _)| *id) + .collect(); + match matching.as_slice() { + [only] => Ok((*only).to_owned()), + _ => Err(DagSelectionError::Input(format!( + "SQLBuild selector name {} matched {} DAG nodes", + python_str_repr(name), + matching.len() + ))), + } +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/constants.rs b/crates/sqlbuild-analysis/src/dag_artifact/constants.rs index 8b877efc20..77a41ef817 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/constants.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/constants.rs @@ -27,3 +27,10 @@ pub(crate) const LOADER_NODE_META_KEY: &str = "sqlbuild_loader_node"; pub(crate) const LOADER_DESTINATION_PREFIX: &str = "__loader__"; /// The most parts a loader destination may have. pub(crate) const LOADER_MAX_PARTS: usize = 3; +pub(crate) const SELECTOR_KIND_NAME: &str = "name"; +pub(crate) const SELECTOR_KIND_TAG: &str = "tag"; +pub(crate) const SELECTOR_KIND_PATH: &str = "path"; +pub(crate) const SELECTOR_KIND_CHECK: &str = "check"; +pub(crate) const SELECTOR_KIND_TEST: &str = "test"; +pub(crate) const SELECTOR_KIND_SEPARATOR: &str = ":"; +pub(crate) const SELECTOR_MODELS_PREFIX: &str = "models/"; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/errors.rs b/crates/sqlbuild-analysis/src/dag_artifact/errors.rs index b37aa7a6eb..461e6c7aa0 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/errors.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/errors.rs @@ -14,3 +14,14 @@ pub enum DagArtifactError { /// The emitter refused the encoded artifact. Emit(JsonEmitError), } + +/// Why a Dagster selection over the DAG artifact failed, in the order Python raised them. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DagSelectionError { + /// The selector token did not parse. + Selector(crate::graph::errors::SelectorError), + /// A `test:` selector, which only `sqb test` and `sqb build` run. + UnitTest(String), + /// `DagsterDagInputError` with this message. + Input(String), +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs index 60780629b8..80514c0a8a 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs @@ -1 +1,2 @@ pub mod build_dag_json; +pub mod resolve_dag_selection; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/resolve_dag_selection.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/resolve_dag_selection.rs new file mode 100644 index 0000000000..2995d66569 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/resolve_dag_selection.rs @@ -0,0 +1,50 @@ +//! Resolve SQLBuild selectors over a DAG artifact into the selected node ids. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::dag_artifact::_helpers::selection::{Adjacency, resolve_expression}; +use crate::dag_artifact::errors::DagSelectionError; +use crate::dag_artifact::models::{DagSelectionGraph, DagSelectionIndex, DagSelectionNode}; + +/// Node ids `select` matches minus those `exclude` matches. +pub fn resolve_dag_selection( + graph: &DagSelectionGraph, + (select, exclude): (&str, Option<&str>), +) -> Result, DagSelectionError> { + let mut nodes: BTreeMap<&str, &DagSelectionNode> = BTreeMap::new(); + for node in &graph.nodes { + nodes.insert(node.id.as_str(), node); + } + let mut upstream: Adjacency = Adjacency::new(); + let mut downstream: Adjacency = Adjacency::new(); + for id in nodes.keys() { + upstream.insert((*id).to_owned(), BTreeSet::new()); + downstream.insert((*id).to_owned(), BTreeSet::new()); + } + for (from_id, to_id) in &graph.edges { + if !nodes.contains_key(from_id.as_str()) || !nodes.contains_key(to_id.as_str()) { + continue; + } + upstream + .entry(to_id.clone()) + .or_default() + .insert(from_id.clone()); + downstream + .entry(from_id.clone()) + .or_default() + .insert(to_id.clone()); + } + let index: DagSelectionIndex<'_> = DagSelectionIndex { + graph, + nodes, + upstream, + downstream, + }; + let mut selected: BTreeSet = resolve_expression(&index, select)?; + if let Some(exclude) = exclude.filter(|exclude| !exclude.is_empty()) { + for excluded in resolve_expression(&index, exclude)? { + selected.remove(&excluded); + } + } + Ok(selected) +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/models.rs b/crates/sqlbuild-analysis/src/dag_artifact/models.rs index 660a118be2..7ec5cf89a3 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/models.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/models.rs @@ -216,3 +216,38 @@ pub struct DagProjectInput { pub target_database: Option, pub target_schema: Option, } + +/// One DAG artifact node as the Dagster selection reads it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DagSelectionNode { + pub id: String, + pub name: String, + pub kind: String, + pub path: Option, + pub tags: Vec, +} + +/// One DAG artifact check: its name, id and checked asset ids. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DagSelectionCheck { + pub name: String, + pub id: String, + pub checked_asset_ids: Vec, +} + +/// The DAG artifact's nodes, `(from, to)` edges and checks. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DagSelectionGraph { + pub nodes: Vec, + pub edges: Vec<(String, String)>, + pub checks: Vec, +} + +/// The node lookup and edge indexes every selector token resolves against. +#[derive(Debug)] +pub(crate) struct DagSelectionIndex<'a> { + pub(crate) graph: &'a DagSelectionGraph, + pub(crate) nodes: std::collections::BTreeMap<&'a str, &'a DagSelectionNode>, + pub(crate) upstream: crate::dag_artifact::_helpers::selection::Adjacency, + pub(crate) downstream: crate::dag_artifact::_helpers::selection::Adjacency, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs index 96eb544955..b90720c229 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod artifact; pub(crate) mod registration; pub(crate) mod rows; +pub(crate) mod selection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs index 29c2baeab9..70ba39f2c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs @@ -2,8 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::dag_artifact::artifact; +use crate::bindings::_helpers::dag_artifact::{artifact, selection}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - artifact::register(module) + artifact::register(module)?; + selection::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs new file mode 100644 index 0000000000..70099991bc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs @@ -0,0 +1,218 @@ +//! Dagster asset selection: SQLBuild selectors resolved over a DAG artifact mapping. + +use pyo3::exceptions::PyOSError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyFrozenSet, PyList, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::dag_artifact::errors::DagSelectionError; +use sqlbuild_analysis::dag_artifact::main::resolve_dag_selection::resolve_dag_selection as resolve; +use sqlbuild_analysis::dag_artifact::models::{ + DagSelectionCheck, DagSelectionGraph, DagSelectionNode, +}; +use sqlbuild_analysis::planner_selection::main::unit_test_selector_rejection::unit_test_selector_rejection; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::planner_selection::carrier::Carrier; + +const DAGSTER_EXCEPTIONS_MODULE: &str = "sqlbuild.integrations.dagster.exceptions"; +const PATHLIB_MODULE: &str = "pathlib"; +const JSON_MODULE: &str = "json"; +const ABC_MODULE: &str = "collections.abc"; +const DAG_VERSION: i64 = 1; +const DAG_LIST_KEYS: [&str; 3] = ["nodes", "edges", "checks"]; + +fn text(value: &Bound<'_, PyAny>) -> PyResult { + value.str()?.extract() +} + +fn field<'py>(mapping: &Bound<'py, PyAny>, key: &str) -> PyResult> { + mapping.call_method1("get", (key,)) +} + +fn items<'py>(mapping: &Bound<'py, PyAny>, key: &str) -> PyResult>> { + let empty: Bound<'py, PyTuple> = PyTuple::empty(mapping.py()); + mapping + .call_method1("get", (key, empty))? + .try_iter()? + .collect() +} + +fn texts(mapping: &Bound<'_, PyAny>, key: &str) -> PyResult> { + items(mapping, key)?.iter().map(text).collect() +} + +fn selection_graph(dag: &Bound<'_, PyAny>) -> PyResult { + let nodes: Vec = items(dag, "nodes")? + .iter() + .map(|node| { + let path: Bound<'_, PyAny> = field(node, "path")?; + Ok(DagSelectionNode { + id: text(&node.get_item("id")?)?, + name: text(&field(node, "name")?)?, + kind: text(&field(node, "kind")?)?, + path: if path.is_instance_of::() { + Some(path.extract()?) + } else { + None + }, + tags: texts(node, "tags")?, + }) + }) + .collect::>()?; + let edges: Vec<(String, String)> = items(dag, "edges")? + .iter() + .map(|edge| { + Ok(( + text(&field(edge, "from_id")?)?, + text(&field(edge, "to_id")?)?, + )) + }) + .collect::>()?; + let checks: Vec = items(dag, "checks")? + .iter() + .map(|check| { + Ok(DagSelectionCheck { + name: text(&field(check, "name")?)?, + id: text(&field(check, "id")?)?, + checked_asset_ids: texts(check, "checked_asset_ids")?, + }) + }) + .collect::>()?; + Ok(DagSelectionGraph { + nodes, + edges, + checks, + }) +} + +fn selection_error(py: Python<'_>, error: DagSelectionError, raw: &str) -> PyErr { + let carrier = match Carrier::single(&PyString::new(py, raw)) { + Ok(carrier) => carrier, + Err(failure) => return failure, + }; + match error { + DagSelectionError::Selector(error) => carrier.error(error), + DagSelectionError::UnitTest(selector) => { + carrier.error(unit_test_selector_rejection(&selector)) + } + DagSelectionError::Input(message) => { + imported(py, DAGSTER_EXCEPTIONS_MODULE, "DagsterDagInputError") + .and_then(|class| class.call1((message,))) + .map_or_else(|failure| failure, PyErr::from_value) + } + } +} + +/// Ids of DAG artifact nodes `select` matches minus those `exclude` matches. +#[pyfunction] +#[pyo3(signature = (*, dag, select, exclude = None))] +fn resolve_dag_selection<'py>( + dag: &Bound<'py, PyAny>, + select: &str, + exclude: Option<&str>, +) -> PyResult> { + let py: Python<'py> = dag.py(); + let graph: DagSelectionGraph = selection_graph(dag)?; + let selected = + resolve(&graph, (select, exclude)).map_err(|error| selection_error(py, error, select))?; + PyFrozenSet::new(py, selected) +} + +fn raised(error_class: &Bound<'_, PyAny>, message: String, cause: Option) -> PyErr { + let py: Python<'_> = error_class.py(); + match error_class.call1((message,)) { + Ok(instance) => { + let error: PyErr = PyErr::from_value(instance); + error.set_cause(py, cause); + error + } + Err(failure) => failure, + } +} + +/// A SQLBuild DAG artifact from a mapping or JSON file path, validated, raising `error_class`. +#[pyfunction] +#[pyo3(signature = (dag, *, error_class))] +fn load_sqlbuild_dag<'py>( + dag: Bound<'py, PyAny>, + error_class: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = dag.py(); + let path_class: Bound<'py, PyAny> = imported(py, PATHLIB_MODULE, "Path")?; + let payload: Bound<'py, PyAny> = if dag.is_instance_of::() + || dag.is_instance(&path_class)? + { + let path: Bound<'py, PyAny> = path_class.call1((&dag,))?; + let encoding: Bound<'py, PyDict> = PyDict::new(py); + encoding.set_item("encoding", "utf-8")?; + let content: Bound<'py, PyAny> = match path.call_method("read_text", (), Some(&encoding)) { + Ok(content) => content, + Err(error) if error.is_instance_of::(py) => { + return Err(raised( + error_class, + format!("could not read SQLBuild DAG artifact: {}", path.str()?), + Some(error), + )); + } + Err(error) => return Err(error), + }; + let json: Bound<'py, PyModule> = py.import(JSON_MODULE)?; + let loaded: Bound<'py, PyAny> = match json.call_method1("loads", (content,)) { + Ok(loaded) => loaded, + Err(error) if error.is_instance(py, &json.getattr("JSONDecodeError")?) => { + let message: String = error.value(py).getattr("msg")?.extract()?; + return Err(raised( + error_class, + format!( + "SQLBuild DAG artifact is not valid JSON: {}: {message}", + path.str()? + ), + Some(error), + )); + } + Err(error) => return Err(error), + }; + if !loaded.is_instance(&imported(py, ABC_MODULE, "Mapping")?)? { + return Err(raised( + error_class, + "SQLBuild DAG artifact must be a JSON object".to_owned(), + None, + )); + } + loaded + } else { + dag + }; + let version: Bound<'py, PyAny> = payload.call_method1("get", ("version",))?; + if version.ne(DAG_VERSION)? { + return Err(raised( + error_class, + format!( + "unsupported SQLBuild DAG artifact version: {}", + version.repr()? + ), + None, + )); + } + for key in DAG_LIST_KEYS { + if !payload + .call_method1("get", (key,))? + .is_instance_of::() + { + return Err(raised( + error_class, + format!("SQLBuild DAG artifact field '{key}' must be a list"), + None, + )); + } + } + Ok(payload) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_dag_selection, module)?)?; + module.add_function(wrap_pyfunction!(load_sqlbuild_dag, module)?)?; + Ok(()) +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 619e0e7b20..b0763e5e7b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4939,6 +4939,12 @@ def aggregate_build_result( source_audit_results: tuple[AuditExecutionResult, ...], end_audit_results: tuple[AuditExecutionResult, ...], ) -> BuildExecutionResult: ... +def resolve_dag_selection( + *, dag: Mapping[str, Any], select: str, exclude: str | None = None +) -> frozenset[str]: ... +def load_sqlbuild_dag( + dag: Mapping[str, Any] | str | Path, *, error_class: type[Exception] +) -> Mapping[str, Any]: ... def fit_artifact_logical_name( *, logical_name: str, fixed_prefix: str, identifier_limit: int, artifact_label: str ) -> str: ... diff --git a/src/sqlbuild/integrations/dagster/_helpers/dag.py b/src/sqlbuild/integrations/dagster/_helpers/dag.py deleted file mode 100644 index 3f6bac8882..0000000000 --- a/src/sqlbuild/integrations/dagster/_helpers/dag.py +++ /dev/null @@ -1,44 +0,0 @@ -"""SQLBuild DAG artifact loading helpers for Dagster.""" - -from __future__ import annotations - -import json -from collections.abc import Mapping -from pathlib import Path -from typing import Any - -from sqlbuild.integrations.dagster.exceptions import DagsterDagInputError -from sqlbuild.integrations.dagster.types import SqlBuildDagInput - - -def load_sqlbuild_dag(dag: SqlBuildDagInput) -> Mapping[str, Any]: - """Load a SQLBuild DAG artifact from a mapping or JSON file path.""" - - if not isinstance(dag, (str, Path)): - payload: Mapping[str, Any] = dag - else: - path: Path = Path(dag) - try: - loaded: Any = json.loads(path.read_text(encoding="utf-8")) - except OSError as error: - raise DagsterDagInputError(f"could not read SQLBuild DAG artifact: {path}") from error - except json.JSONDecodeError as error: - raise DagsterDagInputError( - f"SQLBuild DAG artifact is not valid JSON: {path}: {error.msg}" - ) from error - if not isinstance(loaded, Mapping): - raise DagsterDagInputError("SQLBuild DAG artifact must be a JSON object") - payload = loaded - - _validate_dag_payload(payload) - return payload - - -def _validate_dag_payload(payload: Mapping[str, Any]) -> None: - version: object = payload.get("version") - if version != 1: - raise DagsterDagInputError(f"unsupported SQLBuild DAG artifact version: {version!r}") - for key in ("nodes", "edges", "checks"): - value: object = payload.get(key) - if not isinstance(value, list): - raise DagsterDagInputError(f"SQLBuild DAG artifact field '{key}' must be a list") diff --git a/src/sqlbuild/integrations/dagster/_helpers/selection.py b/src/sqlbuild/integrations/dagster/_helpers/selection.py index 25ac702707..9f935ab8ec 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/selection.py +++ b/src/sqlbuild/integrations/dagster/_helpers/selection.py @@ -5,16 +5,7 @@ from collections.abc import Mapping, Sequence from typing import Any -from sqlbuild._native import ( - parse_project_selector, - path_nodes, - transitive_closure_many, - unit_test_selector_error, -) -from sqlbuild.compiler.planner.constants import SELECTOR_KIND_SEPARATOR -from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector -from sqlbuild.compiler.planner.types import SelectorKind -from sqlbuild.integrations.dagster._helpers.dag import load_sqlbuild_dag +from sqlbuild._native import load_sqlbuild_dag, resolve_dag_selection from sqlbuild.integrations.dagster._helpers.imports import load_dagster from sqlbuild.integrations.dagster.classes.sqlbuild_dagster_translator import ( SqlBuildDagsterTranslator, @@ -36,9 +27,9 @@ def build_sqlbuild_asset_selection_impl( if not sqlbuild_assets: raise DagsterDagInputError("SQLBuild asset selection requires an asset definition") - resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag) + resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag, error_class=DagsterDagInputError) resolved_translator: SqlBuildDagsterTranslator = translator or SqlBuildDagsterTranslator() - selected_ids: set[str] = _resolve_dag_selectors( + selected_ids: frozenset[str] = resolve_dag_selection( dag=resolved_dag, select=sqlbuild_select, exclude=sqlbuild_exclude, @@ -55,185 +46,3 @@ def build_sqlbuild_asset_selection_impl( continue asset_keys.append(resolved_translator.get_asset_key(translated_node)) return load_dagster().AssetSelection.assets(*asset_keys) - - -def _resolve_dag_selectors(*, dag: Mapping[str, Any], select: str, exclude: str | None) -> set[str]: - nodes_by_id: dict[str, Mapping[str, Any]] = { - str(node["id"]): node for node in dag.get("nodes", ()) - } - upstream, downstream = _dependency_indexes(dag=dag, node_ids=set(nodes_by_id)) - selected: set[str] = _resolve_expression( - expression=select, - dag=dag, - nodes_by_id=nodes_by_id, - upstream=upstream, - downstream=downstream, - ) - if exclude: - selected.difference_update( - _resolve_expression( - expression=exclude, - dag=dag, - nodes_by_id=nodes_by_id, - upstream=upstream, - downstream=downstream, - ) - ) - return selected - - -def _resolve_expression( - *, - expression: str, - dag: Mapping[str, Any], - nodes_by_id: Mapping[str, Mapping[str, Any]], - upstream: Mapping[str, set[str]], - downstream: Mapping[str, set[str]], -) -> set[str]: - selected: set[str] = set() - for token in expression.split(): - intersections: list[set[str]] = [ - _resolve_atomic( - raw=part, - dag=dag, - nodes_by_id=nodes_by_id, - upstream=upstream, - downstream=downstream, - ) - for part in token.split(",") - ] - token_selection: set[str] = intersections[0] - for intersection in intersections[1:]: - token_selection.intersection_update(intersection) - selected.update(token_selection) - return selected - - -def _resolve_atomic( - *, - raw: str, - dag: Mapping[str, Any], - nodes_by_id: Mapping[str, Mapping[str, Any]], - upstream: Mapping[str, set[str]], - downstream: Mapping[str, set[str]], -) -> set[str]: - parsed: ParsedSelector | PathSelector = parse_project_selector(raw) - if isinstance(parsed, PathSelector): - start_id: str = _node_id_for_name(nodes_by_id=nodes_by_id, name=parsed.start_name) - end_id: str = _node_id_for_name(nodes_by_id=nodes_by_id, name=parsed.end_name) - matched: set[str] = _path_node_ids(start_id=start_id, end_id=end_id, downstream=downstream) - if parsed.upstream: - matched.update( - transitive_closure_many(starts=(start_id,), edges=upstream, include_starts=True) - ) - if parsed.downstream: - matched.update( - transitive_closure_many(starts=(end_id,), edges=downstream, include_starts=True) - ) - return matched - - matched = _match_parsed_selector(parsed=parsed, dag=dag, nodes_by_id=nodes_by_id) - anchors: frozenset[str] = frozenset(matched) - if parsed.upstream: - matched.update(transitive_closure_many(starts=anchors, edges=upstream, include_starts=True)) - if parsed.downstream: - matched.update( - transitive_closure_many(starts=anchors, edges=downstream, include_starts=True) - ) - return matched - - -def _match_parsed_selector( - *, - parsed: ParsedSelector, - dag: Mapping[str, Any], - nodes_by_id: Mapping[str, Mapping[str, Any]], -) -> set[str]: - if parsed.kind == SelectorKind.TEST: - raise unit_test_selector_error( - selector=f"{SelectorKind.TEST}{SELECTOR_KIND_SEPARATOR}{parsed.value}" - ) - if parsed.kind == SelectorKind.TAG: - return { - node_id - for node_id, node in nodes_by_id.items() - if _node_has_tag(node=node, tag=parsed.value) - } - if parsed.kind == SelectorKind.PATH: - folder: str = parsed.value.replace("\\", "/").strip("/") - return { - node_id - for node_id, node in nodes_by_id.items() - if _path_is_under(path=node.get("path"), folder=folder) - } - if parsed.kind == SelectorKind.CHECK: - checked_ids: set[str] = set() - for check in dag.get("checks", ()): - if str(check.get("name")) == parsed.value or str(check.get("id")) == parsed.value: - checked_ids.update(str(asset_id) for asset_id in check.get("checked_asset_ids", ())) - return checked_ids - expected_kind: str | None = None if parsed.kind == SelectorKind.NAME else str(parsed.kind.value) - return { - node_id - for node_id, node in nodes_by_id.items() - if str(node.get("name")) == parsed.value - and (expected_kind is None or str(node.get("kind")) == expected_kind) - } - - -def _node_has_tag(*, node: Mapping[str, Any], tag: str) -> bool: - return any(str(node_tag) == tag for node_tag in node.get("tags", ())) - - -def _path_is_under(*, path: object, folder: str) -> bool: - if not isinstance(path, str): - return False - normalized_path: str = path.replace("\\", "/").strip("/") - candidates: tuple[str, ...] = ( - folder, - f"models/{folder}" if folder and not folder.startswith("models/") else folder, - ) - return any( - normalized_path == candidate or normalized_path.startswith(f"{candidate}/") - for candidate in candidates - if candidate - ) - - -def _dependency_indexes( - *, dag: Mapping[str, Any], node_ids: set[str] -) -> tuple[dict[str, set[str]], dict[str, set[str]]]: - upstream: dict[str, set[str]] = {node_id: set() for node_id in node_ids} - downstream: dict[str, set[str]] = {node_id: set() for node_id in node_ids} - for edge in dag.get("edges", ()): - from_id: str = str(edge.get("from_id")) - to_id: str = str(edge.get("to_id")) - if from_id not in node_ids or to_id not in node_ids: - continue - upstream[to_id].add(from_id) - downstream[from_id].add(to_id) - return upstream, downstream - - -def _node_id_for_name(*, nodes_by_id: Mapping[str, Mapping[str, Any]], name: str) -> str: - matching_ids: list[str] = [ - node_id for node_id, node in nodes_by_id.items() if str(node.get("name")) == name - ] - if len(matching_ids) != 1: - raise DagsterDagInputError( - f"SQLBuild selector name {name!r} matched {len(matching_ids)} DAG nodes" - ) - return matching_ids[0] - - -def _path_node_ids(*, start_id: str, end_id: str, downstream: Mapping[str, set[str]]) -> set[str]: - on_path: frozenset[str] | None = path_nodes( - start=start_id, - end=end_id, - downstream={node_id: tuple(sorted(targets)) for node_id, targets in downstream.items()}, - ) - if on_path is None: - raise DagsterDagInputError( - f"no SQLBuild DAG path exists between {start_id!r} and {end_id!r}" - ) - return set(on_path) diff --git a/src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_resource.py b/src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_resource.py index 78b53c4544..a20825ec60 100644 --- a/src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_resource.py +++ b/src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_resource.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any -from sqlbuild.integrations.dagster._helpers.dag import load_sqlbuild_dag +from sqlbuild._native import load_sqlbuild_dag from sqlbuild.integrations.dagster._helpers.imports import load_dagster from sqlbuild.integrations.dagster._helpers.invocation_factory import start_sqlbuild_cli_invocation from sqlbuild.integrations.dagster._helpers.translation import translate_sqlbuild_dag @@ -14,6 +14,7 @@ from sqlbuild.integrations.dagster.classes.sqlbuild_dagster_translator import ( SqlBuildDagsterTranslator, ) +from sqlbuild.integrations.dagster.exceptions import DagsterDagInputError from sqlbuild.integrations.dagster.models import SqlBuildProject @@ -58,7 +59,7 @@ def cli( loaded_dag: Mapping[str, Any] | None = None if self.dag_path is not None: loaded_dag = translate_sqlbuild_dag( - dag=load_sqlbuild_dag(self.dag_path), + dag=load_sqlbuild_dag(self.dag_path, error_class=DagsterDagInputError), translator=translator or SqlBuildDagsterTranslator(), ) return start_sqlbuild_cli_invocation( diff --git a/src/sqlbuild/integrations/dagster/main/_sqlbuild_assets.py b/src/sqlbuild/integrations/dagster/main/_sqlbuild_assets.py index 95cfb18a85..160502020c 100644 --- a/src/sqlbuild/integrations/dagster/main/_sqlbuild_assets.py +++ b/src/sqlbuild/integrations/dagster/main/_sqlbuild_assets.py @@ -6,8 +6,8 @@ from pathlib import Path from typing import Any +from sqlbuild._native import load_sqlbuild_dag from sqlbuild.integrations.dagster._helpers.assets import build_asset_specs, build_check_specs -from sqlbuild.integrations.dagster._helpers.dag import load_sqlbuild_dag from sqlbuild.integrations.dagster._helpers.imports import load_dagster from sqlbuild.integrations.dagster.classes.sqlbuild_dagster_translator import ( SqlBuildDagsterTranslator, @@ -40,7 +40,7 @@ def sqlbuild_assets( if project is not None else Path("target/sqlbuild_dag.json") ) - resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag_input) + resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag_input, error_class=DagsterDagInputError) resolved_translator: SqlBuildDagsterTranslator = translator or SqlBuildDagsterTranslator() specs: tuple[Any, ...] = build_asset_specs( dag=resolved_dag, diff --git a/src/sqlbuild/integrations/dagster/main/_sqlbuild_scenario_checks.py b/src/sqlbuild/integrations/dagster/main/_sqlbuild_scenario_checks.py index f43c1e4dc9..dc2c17aba6 100644 --- a/src/sqlbuild/integrations/dagster/main/_sqlbuild_scenario_checks.py +++ b/src/sqlbuild/integrations/dagster/main/_sqlbuild_scenario_checks.py @@ -6,8 +6,8 @@ from pathlib import Path from typing import Any +from sqlbuild._native import load_sqlbuild_dag from sqlbuild.integrations.dagster._helpers.assets import build_scenario_check_specs -from sqlbuild.integrations.dagster._helpers.dag import load_sqlbuild_dag from sqlbuild.integrations.dagster._helpers.imports import load_dagster from sqlbuild.integrations.dagster.classes.sqlbuild_dagster_translator import ( SqlBuildDagsterTranslator, @@ -39,7 +39,7 @@ def sqlbuild_scenario_checks( if project is not None else Path("target/sqlbuild_dag.json") ) - resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag_input) + resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag_input, error_class=DagsterDagInputError) resolved_translator: SqlBuildDagsterTranslator = translator or SqlBuildDagsterTranslator() check_specs: tuple[Any, ...] = build_scenario_check_specs( dag=resolved_dag, diff --git a/src/sqlbuild/integrations/rivers/_helpers/dag.py b/src/sqlbuild/integrations/rivers/_helpers/dag.py deleted file mode 100644 index fc5a33e5ca..0000000000 --- a/src/sqlbuild/integrations/rivers/_helpers/dag.py +++ /dev/null @@ -1,44 +0,0 @@ -"""SQLBuild DAG artifact loading helpers for Rivers.""" - -from __future__ import annotations - -import json -from collections.abc import Mapping -from pathlib import Path -from typing import Any - -from sqlbuild.integrations.rivers.exceptions import RiversDagInputError -from sqlbuild.integrations.rivers.types import SqlBuildDagInput - - -def load_sqlbuild_dag(dag: SqlBuildDagInput) -> Mapping[str, Any]: - """Load a SQLBuild DAG artifact from a mapping or JSON file path.""" - - if not isinstance(dag, (str, Path)): - payload: Mapping[str, Any] = dag - else: - path: Path = Path(dag) - try: - loaded: Any = json.loads(path.read_text(encoding="utf-8")) - except OSError as error: - raise RiversDagInputError(f"could not read SQLBuild DAG artifact: {path}") from error - except json.JSONDecodeError as error: - raise RiversDagInputError( - f"SQLBuild DAG artifact is not valid JSON: {path}: {error.msg}" - ) from error - if not isinstance(loaded, Mapping): - raise RiversDagInputError("SQLBuild DAG artifact must be a JSON object") - payload = loaded - - _validate_dag_payload(payload) - return payload - - -def _validate_dag_payload(payload: Mapping[str, Any]) -> None: - version: object = payload.get("version") - if version != 1: - raise RiversDagInputError(f"unsupported SQLBuild DAG artifact version: {version!r}") - for key in ("nodes", "edges", "checks"): - value: object = payload.get(key) - if not isinstance(value, list): - raise RiversDagInputError(f"SQLBuild DAG artifact field '{key}' must be a list") diff --git a/src/sqlbuild/integrations/rivers/main/_sqlbuild_assets.py b/src/sqlbuild/integrations/rivers/main/_sqlbuild_assets.py index 25adc77073..f2a3d86b4f 100644 --- a/src/sqlbuild/integrations/rivers/main/_sqlbuild_assets.py +++ b/src/sqlbuild/integrations/rivers/main/_sqlbuild_assets.py @@ -6,8 +6,8 @@ from pathlib import Path from typing import Any +from sqlbuild._native import load_sqlbuild_dag from sqlbuild.integrations.rivers._helpers.assets import build_asset_defs -from sqlbuild.integrations.rivers._helpers.dag import load_sqlbuild_dag from sqlbuild.integrations.rivers._helpers.imports import load_rivers from sqlbuild.integrations.rivers.classes.sqlbuild_rivers_translator import ( SqlBuildRiversTranslator, @@ -38,7 +38,7 @@ def sqlbuild_assets( if project is not None else Path("target/sqlbuild_dag.json") ) - resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag_input) + resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag_input, error_class=RiversDagInputError) resolved_translator: SqlBuildRiversTranslator = translator or SqlBuildRiversTranslator() output_defs: tuple[Any, ...] = build_asset_defs( dag=resolved_dag, diff --git a/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py b/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py index a72e977e61..53435d376d 100644 --- a/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py +++ b/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py @@ -11,14 +11,14 @@ import pytest -from sqlbuild._native import PlaygroundCommandRequest +from sqlbuild._native import PlaygroundCommandRequest, load_sqlbuild_dag from sqlbuild._native import run_playground_command as run_playground from sqlbuild.integrations.rivers import SqlBuildProject, sqlbuild_assets from sqlbuild.integrations.rivers._helpers.assets import build_asset_defs -from sqlbuild.integrations.rivers._helpers.dag import load_sqlbuild_dag from sqlbuild.integrations.rivers.classes.sqlbuild_rivers_translator import ( SqlBuildRiversTranslator, ) +from sqlbuild.integrations.rivers.exceptions import RiversDagInputError from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( REPO_ROOT, prepare_waffle_shop, @@ -76,7 +76,7 @@ def sqlbuild_python_nodes(context: Any) -> Iterator[Any]: yield output_defs: tuple[Any, ...] = build_asset_defs( - dag=load_sqlbuild_dag(sqlbuild_project.dag_path), + dag=load_sqlbuild_dag(sqlbuild_project.dag_path, error_class=RiversDagInputError), translator=SqlBuildRiversTranslator(), ) repo: Any = rs.CodeRepository(assets=[sqlbuild_python_nodes]) From 88118d1fa84b1448262300f6689666ce64f36e01 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:08:08 +0100 Subject: [PATCH 06/11] refactor: build the Python-node graph natively around a user-function shim Discovered Python node records, decorator identity configuration, dependency resolution and graph indexes are built natively; Python keeps only the shim that reads provider annotations, identities and decorated dependency names from loaded user functions. Native compile, plan, check, dag and build callers no longer round-trip through Python graph entry points. --- .../_helpers/build_command/execution.rs | 7 +- .../_helpers/build_command/lifecycle.rs | 7 +- .../bindings/_helpers/check/check_command.rs | 11 +- .../_helpers/compile_driver/phases.rs | 11 +- .../_helpers/compile_driver/plan_result.rs | 18 +- .../compile_driver/python_plan_entries.rs | 11 +- .../src/bindings/_helpers/graph/mod.rs | 1 + .../_helpers/graph/python_node_graph.rs | 360 ++++++++++++++++++ .../_helpers/graph/python_selection.rs | 2 +- .../bindings/_helpers/graph/registration.rs | 1 + .../src/bindings/_helpers/graph/views.rs | 2 +- .../bindings/_helpers/query_command/dag.rs | 11 +- python-boundary.toml | 12 +- src/sqlbuild/_native.pyi | 1 + .../python_nodes/_helpers/user_inventory.py | 302 ++------------- .../compiler/python_nodes/main/_graph.py | 15 - .../python_nodes/main/_run_selection.py | 29 -- .../compiler/python_nodes/_helpers/helpers.py | 7 +- .../python_nodes/_helpers/test_inventory.py | 115 +++--- .../executor/python_nodes/_helpers/helpers.py | 3 +- 20 files changed, 472 insertions(+), 454 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs delete mode 100644 src/sqlbuild/compiler/python_nodes/main/_graph.py delete mode 100644 src/sqlbuild/compiler/python_nodes/main/_run_selection.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index a35915276a..7b59a30828 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -20,6 +20,7 @@ use crate::bindings::_helpers::check::check_core::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; const NATIVE_MODULE: &str = "sqlbuild._native"; const CURSOR_OVERRIDES_MODULE: &str = "sqlbuild._native"; @@ -28,7 +29,6 @@ const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const PIPELINE_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const FRESHNESS_MODULE: &str = "sqlbuild.cli.commands._helpers.freshness.source_freshness"; const COST_CONTEXT_MODULE: &str = "sqlbuild._native"; -const GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const CHECKS_MODULE: &str = "sqlbuild.executor.python_nodes.main._checks"; const EXECUTOR_PYTHON_MODELS: &str = "sqlbuild.executor.python_nodes.models"; const EXECUTION_COMMAND: &str = "sqb build"; @@ -493,10 +493,7 @@ pub(crate) fn run_post_build_python_checks<'py>( let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?; let plan: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; let load_results: Bound<'py, PyAny> = result.getattr("load_results")?; - let graph: Bound<'py, PyAny> = call_with( - &imported(py, GRAPH_MODULE, "build_discovered_python_node_graph")?, - &[("discovered_inputs", discovered.clone())], - )?; + let graph: Bound<'py, PyAny> = python_node_graph(&discovered)?; let by_loader = || -> PyResult> { Ok(load_results_by_loader_name(&plan.getattr("source_map")?, &load_results)?.into_any()) }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index 122d228eeb..bae571a778 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -15,9 +15,9 @@ use crate::bindings::_helpers::build_command::support::{ boolean, emit, frozenset, or_none, text, tuple, }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; const NATIVE_MODULE: &str = "sqlbuild._native"; -const GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const PYTHON_NODE_MODELS: &str = "sqlbuild.compiler.python_nodes.models"; const LOADER_DAG_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; @@ -299,10 +299,7 @@ fn lifecycle_plan<'py>( ) -> PyResult> { let py: Python<'py> = discovered.py(); let plan_output: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; - let graph: Bound<'py, PyAny> = call_with( - &imported(py, GRAPH_MODULE, "build_discovered_python_node_graph")?, - &[("discovered_inputs", discovered.clone())], - )?; + let graph: Bound<'py, PyAny> = python_node_graph(discovered)?; let selection: Bound<'py, PyAny> = call_with( &imported(py, PYTHON_NODE_MODELS, "PythonSqlRunSelection")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index eb7034d6d9..59b75fb7af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -22,6 +22,7 @@ use crate::bindings::_helpers::check::check_results::{ use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::compile_outputs::runtime_target::write_python_check_runtime_target; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::graph::python_selection::python_node_names; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -35,7 +36,6 @@ const SELECTOR_PARSE_MODULE: &str = "sqlbuild._native"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; -const PYTHON_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const PYTHON_NODE_MODELS: &str = "sqlbuild.compiler.python_nodes.models"; const PYTHON_NODE_TYPES: &str = "sqlbuild.compiler.python_nodes.types"; const EXECUTOR_MODELS: &str = "sqlbuild.executor.python_nodes.models"; @@ -291,14 +291,7 @@ fn python_selection<'py>( discovered: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyFrozenSet>)> { let py: Python<'py> = request.py(); - let python_graph: Bound<'py, PyAny> = call_with( - &imported( - py, - PYTHON_GRAPH_MODULE, - "build_discovered_python_node_graph", - )?, - &[("discovered_inputs", discovered.clone())], - )?; + let python_graph: Bound<'py, PyAny> = python_node_graph(discovered)?; call_with( &imported(py, REJECT_UNIT_TESTS_MODULE, "reject_unit_test_selectors")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index c407dc3e15..2a3cf543f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -11,10 +11,10 @@ use crate::bindings::_helpers::compile_command::target_writer::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{Status, call_with, imported}; use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const PYTHON_NODES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; @@ -113,14 +113,7 @@ pub(crate) fn write_dag_artifact<'py>( let started: f64 = monotonic(py)?; status.start("Writing DAG artifact...")?; let discovered: Bound<'py, PyAny> = analysis.getattr("discovered_inputs")?; - let python_graph: Bound<'py, PyAny> = call_with( - &imported( - py, - PYTHON_NODES_MODULE, - "build_discovered_python_node_graph", - )?, - &[("discovered_inputs", discovered.clone())], - )?; + let python_graph: Bound<'py, PyAny> = python_node_graph(&discovered)?; let path: Bound<'py, PyAny> = resolved_dag_path(project_dir, dag_path)?; let project_name: String = discovered .getattr("project_config")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 2b188c660f..19b65a0be8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -9,12 +9,12 @@ use crate::bindings::_helpers::compile_driver::phases::monotonic; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::graph::python_node_graph::run_selection_from_inputs; const NATIVE_MODULE: &str = "sqlbuild._native"; const EXECUTION_PLAN_MODULE: &str = "sqlbuild._native"; const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const MATERIALIZATIONS_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.materializations"; -const RUN_SELECTION_MODULE: &str = "sqlbuild.compiler.python_nodes.main._run_selection"; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const SINGLE_CONNECTION: i64 = 1; @@ -58,18 +58,10 @@ pub(crate) fn pipeline_result<'py>( &native.getattr("build_project_graph_from_compiled_project")?, &[("project", project.clone())], )?; - run_selection = call_with( - &imported( - py, - RUN_SELECTION_MODULE, - "resolve_python_sql_run_selection_from_inputs", - )?, - &[ - ("select", select.clone()), - ("exclude", exclude.clone()), - ("project_graph", graph), - ("discovered_inputs", discovered.clone()), - ], + run_selection = run_selection_from_inputs( + (select.extract()?, exclude.extract()?), + &graph, + &discovered, )?; selected_sql_keys = run_selection.getattr("sql_keys")?; selected_python_node_names = run_selection.getattr("python_node_names")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs index 403eda0b72..8491a94905 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs @@ -12,9 +12,9 @@ use sqlbuild_analysis::python_plan::main::upstream_closure::upstream_closure; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; -const NODE_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const NATIVE_MODULE: &str = "sqlbuild._native"; const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.compiler.python_nodes.models"; const PYTHON_NODE_TYPES_MODULE: &str = "sqlbuild.compiler.python_nodes.types"; @@ -63,14 +63,7 @@ fn upstream_names(graph: &Bound<'_, PyAny>) -> PyResult(discovered: &Bound<'py, PyAny>) -> PyResult> { - call_with( - &imported( - discovered.py(), - NODE_GRAPH_MODULE, - "build_discovered_python_node_graph", - )?, - &[("discovered_inputs", discovered.clone())], - ) + python_node_graph(discovered) } /// Return sources declared by the selected Python nodes and their Python upstreams. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs index e33df4195c..af9d80ac3c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs @@ -3,6 +3,7 @@ mod execution_gates; pub(crate) mod keys; pub(crate) mod project_graph; mod python_lifecycle; +pub(crate) mod python_node_graph; pub(crate) mod python_selection; pub(crate) mod registration; pub(crate) mod views; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs new file mode 100644 index 0000000000..fa8f0d1cee --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs @@ -0,0 +1,360 @@ +//! The executable Python-node graph built from discovered project inputs. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; +use sqlbuild_analysis::python_selection::models::{InventoryIndexes, InventoryNode}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::graph::python_selection::resolve_python_sql_run_selection; +use crate::bindings::_helpers::graph::views::compiled_graph_keys; + +const USER_INVENTORY_MODULE: &str = "sqlbuild.compiler.python_nodes._helpers.user_inventory"; +const NODE_MODELS_MODULE: &str = "sqlbuild.compiler.python_nodes.models"; +const NODE_TYPES_MODULE: &str = "sqlbuild.compiler.python_nodes.types"; +const SQL_REF_MODULE: &str = "sqlbuild.python_nodes.models"; +const NAME_KEY: &str = "name"; + +/// One discovered function collection, its node kind and whether it carries documentation. +const KINDS: [(&str, &str, bool); 4] = [ + ("loader_functions", "LOADER", false), + ("task_functions", "TASK", true), + ("asset_functions", "ASSET", true), + ("check_functions", "CHECK", true), +]; + +/// Python classes and shims every node is built with. +struct Vocabulary<'py> { + node: Bound<'py, PyAny>, + kinds: Bound<'py, PyAny>, + sql_ref: Bound<'py, PyAny>, + user_facts: Bound<'py, PyAny>, + providers: Bound<'py, PyDict>, +} + +fn attribute_value<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult> { + value.getattr(name) +} + +fn optional_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(value.clone()); + } + value.getattr("value") +} + +/// The kind-specific decorator configuration and metadata record of one discovered function. +fn kind_parts<'py>( + py: Python<'py>, + kind: &str, + discovered: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyDict>, &'static str, Bound<'py, PyAny>)> { + let config: Bound<'py, PyDict> = PyDict::new(py); + let get = |name: &str| attribute_value(discovered, name); + let metadata = |class: &str, fields: &[&str]| -> PyResult> { + let pairs: Vec<(&str, Bound<'py, PyAny>)> = fields + .iter() + .map(|field| Ok((*field, get(field)?))) + .collect::>()?; + call_with(&imported(py, NODE_MODELS_MODULE, class)?, &pairs) + }; + Ok(match kind { + "LOADER" => { + config.set_item("columns", get("columns")?)?; + config.set_item("contract", get("contract")?)?; + config.set_item("cursor_column", get("cursor_column")?)?; + config.set_item("destination", get("destination")?)?; + config.set_item("unique_key", get("unique_key")?)?; + config.set_item("write_strategy", optional_value(&get("write_strategy")?)?)?; + let record = metadata( + "DiscoveredPythonLoaderMetadata", + &[ + "destination", + "write_strategy", + "cursor_column", + "unique_key", + "columns", + "contract", + "connection_mode", + ], + )?; + (config, "loader", record) + } + "TASK" => ( + config, + "task", + metadata("DiscoveredPythonTaskMetadata", &["retry"])?, + ), + "ASSET" => { + config.set_item("columns", get("columns")?)?; + config.set_item("column_lineage", get("column_lineage")?)?; + let record = metadata( + "DiscoveredPythonAssetMetadata", + &["columns", "column_lineage", "retry"], + )?; + (config, "asset", record) + } + _ => { + config.set_item("severity", get("severity")?.getattr("value")?)?; + let record = metadata("DiscoveredPythonCheckMetadata", &["severity"])?; + (config, "check", record) + } + }) +} + +/// One `DiscoveredPythonNode` with its identity and provider usages read by the user shim. +fn python_node<'py>( + vocabulary: &Vocabulary<'py>, + (kind, documented): (&str, bool), + discovered: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let get = |name: &str| attribute_value(discovered, name); + let (config, metadata_field, metadata) = kind_parts(py, kind, discovered)?; + if documented { + for field in ["description", "group", "meta", "tags"] { + config.set_item(field, get(field)?)?; + } + } + let kind_member: Bound<'py, PyAny> = vocabulary.kinds.getattr(kind)?; + let (usages, identity): (Bound<'py, PyAny>, Bound<'py, PyAny>) = call_with( + &vocabulary.user_facts, + &[ + ("kind", kind_member.getattr("value")?), + ("name", get("name")?), + ("function", get("function")?), + ("file_path", get("file_path")?), + ("provider_by_name", vocabulary.providers.clone().into_any()), + ("decorator_config", config.into_any()), + ], + )? + .extract()?; + let depends_on: Bound<'py, PyAny> = get("depends_on")?; + let mut sql_deps: Vec> = Vec::new(); + for dependency in depends_on.try_iter()? { + let dependency: Bound<'py, PyAny> = dependency?; + if dependency.is_instance(&vocabulary.sql_ref)? { + sql_deps.push(dependency); + } + } + let mut pairs: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("kind", kind_member), + ("file_path", get("file_path")?), + ("relative_path", get("relative_path")?), + ("name", get("name")?), + ("function", get("function")?), + ("depends_on", depends_on), + ("sql_deps", PyTuple::new(py, sql_deps)?.into_any()), + ("provider_usages", usages), + ("identity", identity), + ("description", get("description")?), + (metadata_field, metadata), + ]; + if documented { + for field in ["tags", "group", "meta"] { + pairs.push((field, get(field)?)); + } + } + call_with(&vocabulary.node, &pairs) +} + +/// Each node's upstream Python-node names, resolving decorated dependencies through the shim. +fn upstream_names( + py: Python<'_>, + nodes: &[Bound<'_, PyAny>], + sql_ref: &Bound<'_, PyAny>, +) -> PyResult>> { + let by_key: Bound<'_, PyDict> = PyDict::new(py); + for node in nodes { + by_key.set_item(node.getattr("function")?, node.getattr("name")?)?; + } + for node in nodes { + let name: Bound<'_, PyAny> = node.getattr("name")?; + by_key.set_item( + PyTuple::new(py, [PyString::new(py, NAME_KEY).into_any(), name.clone()])?, + name, + )?; + } + let dependency_key: Bound<'_, PyAny> = + imported(py, USER_INVENTORY_MODULE, "python_node_dependency_key")?; + nodes + .iter() + .map(|node| { + let mut names: Vec = Vec::new(); + for dependency in node.getattr("depends_on")?.try_iter()? { + let dependency: Bound<'_, PyAny> = dependency?; + if dependency.is_instance(sql_ref)? { + continue; + } + let key: Bound<'_, PyAny> = dependency_key.call1((dependency,))?; + if let Some(upstream) = by_key.get_item(key)? { + names.push(upstream.extract()?); + } + } + Ok(names) + }) + .collect() +} + +fn inventory_node(node: &Bound<'_, PyAny>, upstream: Vec) -> PyResult { + Ok(InventoryNode { + name: node.getattr("name")?.extract()?, + tags: node.getattr("tags")?.extract()?, + relative_path: node + .getattr("relative_path")? + .call_method0("as_posix")? + .extract()?, + upstream, + }) +} + +/// The internal executable Python-node graph of discovered project inputs. +pub(crate) fn python_node_graph<'py>( + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered_inputs.py(); + let providers: Bound<'py, PyDict> = PyDict::new(py); + for provider in discovered_inputs.getattr("providers")?.try_iter()? { + let provider: Bound<'py, PyAny> = provider?; + providers.set_item(provider.getattr("name")?, provider)?; + } + let vocabulary: Vocabulary<'py> = Vocabulary { + node: imported(py, NODE_MODELS_MODULE, "DiscoveredPythonNode")?, + kinds: imported(py, NODE_TYPES_MODULE, "PythonNodeKind")?, + sql_ref: imported(py, SQL_REF_MODULE, "SqlResourceRef")?, + user_facts: imported(py, USER_INVENTORY_MODULE, "python_node_user_facts")?, + providers, + }; + let mut nodes: Vec> = Vec::new(); + for (collection, kind, documented) in KINDS { + for discovered in discovered_inputs.getattr(collection)?.try_iter()? { + nodes.push(python_node(&vocabulary, (kind, documented), &discovered?)?); + } + } + let inventory: Vec = nodes + .iter() + .zip(upstream_names(py, &nodes, &vocabulary.sql_ref)?) + .map(|(node, upstream)| inventory_node(node, upstream)) + .collect::>()?; + let InventoryIndexes { + edges, + upstream, + downstream, + tags, + paths, + } = graph_indexes(&inventory); + graph_record(py, &nodes, (edges, upstream, downstream, tags, paths)) +} + +type NameRows = Vec<(String, Vec)>; + +fn graph_record<'py>( + py: Python<'py>, + nodes: &[Bound<'py, PyAny>], + (edges, upstream, downstream, tags, paths): ( + Vec<(String, String)>, + NameRows, + NameRows, + NameRows, + Vec<(String, String)>, + ), +) -> PyResult> { + let edge_class: Bound<'py, PyAny> = + imported(py, NODE_MODELS_MODULE, "PythonNodeDependencyEdge")?; + let dependency_edges: Vec> = edges + .into_iter() + .map(|(upstream_name, downstream_name)| { + call_with( + &edge_class, + &[ + ( + "upstream_name", + PyString::new(py, &upstream_name).into_any(), + ), + ( + "downstream_name", + PyString::new(py, &downstream_name).into_any(), + ), + ], + ) + }) + .collect::>()?; + let adjacency = |rows: NameRows| -> PyResult> { + let mapping: Bound<'py, PyDict> = PyDict::new(py); + for (name, names) in rows { + mapping.set_item(name, PyTuple::new(py, names)?)?; + } + Ok(mapping) + }; + let tag_index: Bound<'py, PyDict> = PyDict::new(py); + for (tag, names) in tags { + tag_index.set_item(tag, PyFrozenSet::new(py, names)?)?; + } + let path_index: Bound<'py, PyDict> = PyDict::new(py); + for (name, path) in paths { + path_index.set_item(name, path)?; + } + let by_name: Bound<'py, PyDict> = PyDict::new(py); + let by_selector: Bound<'py, PyDict> = PyDict::new(py); + for node in nodes { + let name: Bound<'py, PyAny> = node.getattr("name")?; + let kind: String = node.getattr("kind")?.getattr("value")?.extract()?; + by_name.set_item(&name, node)?; + by_selector.set_item(format!("{kind}:{}", name.str()?), node)?; + } + call_with( + &imported(py, NODE_MODELS_MODULE, "PythonNodeGraph")?, + &[ + ("nodes", PyTuple::new(py, nodes)?.into_any()), + ( + "dependency_edges", + PyTuple::new(py, dependency_edges)?.into_any(), + ), + ("upstream_deps", adjacency(upstream)?.into_any()), + ("downstream_deps", adjacency(downstream)?.into_any()), + ("tag_index", tag_index.into_any()), + ("path_index", path_index.into_any()), + ("nodes_by_name", by_name.into_any()), + ("nodes_by_typed_selector", by_selector.into_any()), + ], + ) +} + +/// `PythonSqlRunSelection` of run selectors over the project graph and discovered Python nodes. +pub(crate) fn run_selection_from_inputs<'py>( + (select, exclude): (Vec, Vec), + project_graph: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_graph.py(); + let python_graph: Bound<'py, PyAny> = python_node_graph(discovered_inputs)?; + let (sql_keys, python_node_names) = + resolve_python_sql_run_selection(project_graph, &python_graph, select, exclude)?; + call_with( + &imported(py, NODE_MODELS_MODULE, "PythonSqlRunSelection")?, + &[ + ("sql_keys", compiled_graph_keys(py, sql_keys)?.into_any()), + ( + "python_node_names", + PyFrozenSet::new(py, python_node_names)?.into_any(), + ), + ], + ) +} + +/// Build the internal executable Python-node graph from discovered project inputs. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_python_node_graph<'py>( + options: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("build_python_node_graph", options)?; + python_node_graph(&request.get("discovered_inputs")?) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_python_node_graph, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index b61ae69be3..7cb087cb02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -160,7 +160,7 @@ fn resolve_python_check_excludes<'py>( /// `(sql keys, python node names)` run selectors resolve to across SQL resources and Python nodes. #[pyfunction] #[pyo3(signature = (*, project_graph, python_graph, select, exclude))] -fn resolve_python_sql_run_selection( +pub(crate) fn resolve_python_sql_run_selection( project_graph: &Bound<'_, PyAny>, python_graph: &Bound<'_, PyAny>, select: Vec, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs index f9e662f18c..971b8975e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs @@ -6,5 +6,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::graph::execution_gates::register(module)?; crate::bindings::_helpers::graph::python_selection::register(module)?; crate::bindings::_helpers::graph::python_lifecycle::register(module)?; + crate::bindings::_helpers::graph::python_node_graph::register(module)?; crate::bindings::_helpers::graph::views::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs index 01218702a9..79a3b3be7e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs @@ -29,7 +29,7 @@ fn native_graph_keys(keys: &Bound<'_, PyAny>) -> PyResult> { /// `CompiledObjectKey`s for native graph key pairs. #[pyfunction] -fn compiled_graph_keys<'py>( +pub(crate) fn compiled_graph_keys<'py>( py: Python<'py>, keys: Vec, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs index 9f2e10cf29..73b24b16fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs @@ -9,11 +9,11 @@ use sqlbuild_core::query_command::main::render_dag_summary::render_dag_summary; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::query_command::project::{ProjectAdapter, project_adapter}; const NATIVE_MODULE: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; -const PYTHON_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; fn array_length(document: &serde_json::Value, key: &str) -> PyResult { document @@ -69,14 +69,7 @@ fn run_dag_command<'py>(py: Python<'py>, options: Option>) -> ("external_sql_reference_resolver", resolver), ], )?; - let python_graph: Bound<'py, PyAny> = call_with( - &imported( - py, - PYTHON_GRAPH_MODULE, - "build_discovered_python_node_graph", - )?, - &[("discovered_inputs", discovered.clone())], - )?; + let python_graph: Bound<'py, PyAny> = python_node_graph(&discovered)?; let dag_json: String = call_with( &imported(py, NATIVE_MODULE, "build_dag_json")?, &[ diff --git a/python-boundary.toml b/python-boundary.toml index 493b802784..1a723fac71 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -402,7 +402,7 @@ note = "Tracks the loaded user modules macro call results depend on, and runs a [[stays]] path = "compiler/python_nodes/_helpers/user_inventory.py" reason = "user-code" -note = "Builds Python nodes from loaded user functions: signatures, provider annotations and decorated dependencies." +note = "Reads loaded user node functions: provider parameter annotations, identities over live functions and decorated dependency names; graph construction is native." [[stays]] path = "compiler/discovery/_helpers/filesystem/core.py" @@ -464,16 +464,6 @@ path = "compiler/python_nodes/main/_hook_identities.py" reason = "user-code" note = "Builds identities of loaded user hook functions." -[[stays]] -path = "compiler/python_nodes/main/_graph.py" -reason = "user-code" -note = "Builds the Python node graph from loaded user functions." - -[[stays]] -path = "compiler/python_nodes/main/_run_selection.py" -reason = "user-code" -note = "Builds the Python node graph from loaded user functions for run selection." - [[stays]] path = "compiler/auditing/main/above.py" reason = "public-api" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b0763e5e7b..c5d57dae9a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4945,6 +4945,7 @@ def resolve_dag_selection( def load_sqlbuild_dag( dag: Mapping[str, Any] | str | Path, *, error_class: type[Exception] ) -> Mapping[str, Any]: ... +def build_python_node_graph(*, discovered_inputs: DiscoveredProjectInputs) -> PythonNodeGraph: ... def fit_artifact_logical_name( *, logical_name: str, fixed_prefix: str, identifier_limit: int, artifact_label: str ) -> str: ... diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py b/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py index 9a94fa2409..c16204c3a8 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py @@ -1,34 +1,15 @@ -"""Python-node inventory over loaded user functions: signatures, providers and dependencies.""" +"""Python-node facts read from loaded user functions: identities, providers and dependencies.""" from __future__ import annotations import inspect from collections.abc import Callable -from dataclasses import replace from pathlib import Path -from typing import Protocol, get_type_hints +from typing import get_type_hints -from sqlbuild._native import python_node_graph_indexes -from sqlbuild.compiler.discovery.models import ( - DiscoveredAssetFunction, - DiscoveredCheckFunction, - DiscoveredLoaderFunction, - DiscoveredProjectInputs, - DiscoveredProvider, - DiscoveredProviderUsage, - DiscoveredTaskFunction, -) +from sqlbuild.compiler.discovery.models import DiscoveredProvider, DiscoveredProviderUsage from sqlbuild.compiler.python_nodes._helpers.identity import build_python_identity -from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonAssetMetadata, - DiscoveredPythonCheckMetadata, - DiscoveredPythonLoaderMetadata, - DiscoveredPythonNode, - DiscoveredPythonTaskMetadata, - PythonNodeDependencyEdge, - PythonNodeGraph, -) -from sqlbuild.compiler.python_nodes.types import PythonNodeKind +from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity from sqlbuild.provider.classes.provider import Provider from sqlbuild.python_nodes.main.read_asset_definition import read_asset_definition from sqlbuild.python_nodes.main.read_check_definition import read_check_definition @@ -38,265 +19,38 @@ AssetDefinition, CheckDefinition, LoaderDefinition, - SqlResourceRef, TaskDefinition, ) -def build_python_node_graph(*, discovered_inputs: DiscoveredProjectInputs) -> PythonNodeGraph: - """Build the internal executable Python-node graph from discovered project inputs.""" - - nodes: tuple[DiscoveredPythonNode, ...] = build_python_nodes( - loader_functions=discovered_inputs.loader_functions, - task_functions=discovered_inputs.task_functions, - asset_functions=discovered_inputs.asset_functions, - check_functions=discovered_inputs.check_functions, - providers=discovered_inputs.providers, - ) - edges, upstream, downstream, tags, paths = python_node_graph_indexes( - [ - (node.name, node.tags, node.relative_path.as_posix(), upstream_names) - for node, upstream_names in zip(nodes, _upstream_node_names(nodes), strict=True) - ] - ) - return PythonNodeGraph( - nodes=nodes, - dependency_edges=tuple( - PythonNodeDependencyEdge(upstream_name=upstream_name, downstream_name=downstream_name) - for upstream_name, downstream_name in edges - ), - upstream_deps={name: tuple(names) for name, names in upstream}, - downstream_deps={name: tuple(names) for name, names in downstream}, - tag_index={tag: frozenset(names) for tag, names in tags}, - path_index=dict(paths), - nodes_by_name={node.name: node for node in nodes}, - nodes_by_typed_selector={f"{node.kind.value}:{node.name}": node for node in nodes}, - ) - - -def build_python_nodes( - *, - loader_functions: tuple[DiscoveredLoaderFunction, ...] = (), - task_functions: tuple[DiscoveredTaskFunction, ...] = (), - asset_functions: tuple[DiscoveredAssetFunction, ...] = (), - check_functions: tuple[DiscoveredCheckFunction, ...] = (), - providers: tuple[DiscoveredProvider, ...] = (), -) -> tuple[DiscoveredPythonNode, ...]: - """Return the shared internal node view for all executable Python node kinds.""" - - nodes: list[DiscoveredPythonNode] = [] - provider_by_name: dict[str, DiscoveredProvider] = { - discovered_provider.name: discovered_provider for discovered_provider in providers - } - loader_function: DiscoveredLoaderFunction - for loader_function in loader_functions: - nodes.append(_build_loader_node(loader=loader_function, provider_by_name=provider_by_name)) - task_function: DiscoveredTaskFunction - for task_function in task_functions: - nodes.append(_build_task_node(task=task_function, provider_by_name=provider_by_name)) - asset_function: DiscoveredAssetFunction - for asset_function in asset_functions: - nodes.append(_build_asset_node(asset=asset_function, provider_by_name=provider_by_name)) - check_function: DiscoveredCheckFunction - for check_function in check_functions: - nodes.append(_build_check_node(check=check_function, provider_by_name=provider_by_name)) - return tuple(nodes) - - -def _upstream_node_names(nodes: tuple[DiscoveredPythonNode, ...]) -> list[list[str]]: - """Each node's Python-node dependencies, resolving live callables to node names.""" - - node_by_dependency_key: dict[object | tuple[str, str], DiscoveredPythonNode] = { - node.function: node for node in nodes - } - node: DiscoveredPythonNode - for node in nodes: - node_by_dependency_key[("name", node.name)] = node - upstream_names: list[list[str]] = [] - for node in nodes: - names: list[str] = [] - dependency: Callable[..., object] | SqlResourceRef - for dependency in node.depends_on: - if isinstance(dependency, SqlResourceRef): - continue - upstream_node: DiscoveredPythonNode | None = node_by_dependency_key.get( - _python_node_dependency_key(dependency) - ) - if upstream_node is not None: - names.append(upstream_node.name) - upstream_names.append(names) - return upstream_names - - -class _DiscoveredPythonFunction(Protocol): - @property - def file_path(self) -> Path: ... - - @property - def relative_path(self) -> Path: ... - - @property - def name(self) -> str: ... - - @property - def function(self) -> Callable[..., object]: ... - - @property - def depends_on(self) -> tuple[Callable[..., object] | SqlResourceRef, ...]: ... - - -class _DocumentedPythonFunction(_DiscoveredPythonFunction, Protocol): - @property - def tags(self) -> tuple[str, ...]: ... - - @property - def group(self) -> str | None: ... - - @property - def description(self) -> str | None: ... - - @property - def meta(self) -> dict[str, object] | None: ... - - -def _build_loader_node( - *, loader: DiscoveredLoaderFunction, provider_by_name: dict[str, DiscoveredProvider] -) -> DiscoveredPythonNode: - return replace( - _build_python_node( - kind=PythonNodeKind.LOADER, - discovered=loader, - provider_by_name=provider_by_name, - decorator_config={ - "columns": loader.columns, - "contract": loader.contract, - "cursor_column": loader.cursor_column, - "destination": loader.destination, - "unique_key": loader.unique_key, - "write_strategy": loader.write_strategy.value - if loader.write_strategy is not None - else None, - }, - ), - description=loader.description, - loader=DiscoveredPythonLoaderMetadata( - destination=loader.destination, - write_strategy=loader.write_strategy, - cursor_column=loader.cursor_column, - unique_key=loader.unique_key, - columns=loader.columns, - contract=loader.contract, - connection_mode=loader.connection_mode, - ), - ) - - -def _build_task_node( - *, task: DiscoveredTaskFunction, provider_by_name: dict[str, DiscoveredProvider] -) -> DiscoveredPythonNode: - return replace( - _build_documented_python_node( - kind=PythonNodeKind.TASK, - discovered=task, - provider_by_name=provider_by_name, - decorator_config={}, - ), - task=DiscoveredPythonTaskMetadata(retry=task.retry), - ) - - -def _build_asset_node( - *, asset: DiscoveredAssetFunction, provider_by_name: dict[str, DiscoveredProvider] -) -> DiscoveredPythonNode: - return replace( - _build_documented_python_node( - kind=PythonNodeKind.ASSET, - discovered=asset, - provider_by_name=provider_by_name, - decorator_config={"columns": asset.columns, "column_lineage": asset.column_lineage}, - ), - asset=DiscoveredPythonAssetMetadata( - columns=asset.columns, - column_lineage=asset.column_lineage, - retry=asset.retry, - ), - ) - - -def _build_check_node( - *, check: DiscoveredCheckFunction, provider_by_name: dict[str, DiscoveredProvider] -) -> DiscoveredPythonNode: - return replace( - _build_documented_python_node( - kind=PythonNodeKind.CHECK, - discovered=check, - provider_by_name=provider_by_name, - decorator_config={"severity": check.severity.value}, - ), - check=DiscoveredPythonCheckMetadata(severity=check.severity), - ) - - -def _build_documented_python_node( - *, - kind: PythonNodeKind, - discovered: _DocumentedPythonFunction, - provider_by_name: dict[str, DiscoveredProvider], - decorator_config: dict[str, object], -) -> DiscoveredPythonNode: - return replace( - _build_python_node( - kind=kind, - discovered=discovered, - provider_by_name=provider_by_name, - decorator_config={ - **decorator_config, - "description": discovered.description, - "group": discovered.group, - "meta": discovered.meta, - "tags": discovered.tags, - }, - ), - tags=discovered.tags, - group=discovered.group, - description=discovered.description, - meta=discovered.meta, - ) - - -def _build_python_node( +def python_node_user_facts( *, - kind: PythonNodeKind, - discovered: _DiscoveredPythonFunction, + kind: str, + name: str, + function: Callable[..., object], + file_path: Path, provider_by_name: dict[str, DiscoveredProvider], decorator_config: dict[str, object], -) -> DiscoveredPythonNode: - """Build the kind-independent node view; decorator_config is exactly the identity input.""" - - return DiscoveredPythonNode( - kind=kind, - file_path=discovered.file_path, - relative_path=discovered.relative_path, - name=discovered.name, - function=discovered.function, - depends_on=discovered.depends_on, - sql_deps=_sql_deps(discovered.depends_on), - provider_usages=_provider_usages( - function=discovered.function, provider_by_name=provider_by_name - ), - identity=build_python_identity( - node_type=kind.value, - node_name=discovered.name, - function=discovered.function, - project_dir=discovered.file_path.parent, +) -> tuple[tuple[DiscoveredProviderUsage, ...], PythonNodeIdentity]: + """Read provider parameters and the identity of one loaded user node function.""" + + return ( + provider_usages(function=function, provider_by_name=provider_by_name), + build_python_identity( + node_type=kind, + node_name=name, + function=function, + project_dir=file_path.parent, decorator_config=decorator_config, ), ) -def _provider_usages( +def provider_usages( *, function: Callable[..., object], provider_by_name: dict[str, DiscoveredProvider] ) -> tuple[DiscoveredProviderUsage, ...]: + """Provider parameters of one loaded user function and their annotation classes.""" + usages: list[DiscoveredProviderUsage] = [] type_hints: dict[str, object] = get_type_hints(function) parameter: inspect.Parameter @@ -321,7 +75,9 @@ def _provider_usages( return tuple(usages) -def _python_node_dependency_key(dependency: object) -> object | tuple[str, str]: +def python_node_dependency_key(dependency: object) -> object | tuple[str, str]: + """The node name a decorated dependency declares, or the dependency itself.""" + loader_definition: LoaderDefinition | None = ( read_loader_definition(dependency) if callable(dependency) else None ) @@ -343,11 +99,3 @@ def _python_node_dependency_key(dependency: object) -> object | tuple[str, str]: if check_definition is not None: return ("name", check_definition.name) return dependency - - -def _sql_deps( - dependencies: tuple[Callable[..., object] | SqlResourceRef, ...], -) -> tuple[SqlResourceRef, ...]: - return tuple( - dependency for dependency in dependencies if isinstance(dependency, SqlResourceRef) - ) diff --git a/src/sqlbuild/compiler/python_nodes/main/_graph.py b/src/sqlbuild/compiler/python_nodes/main/_graph.py deleted file mode 100644 index b968382904..0000000000 --- a/src/sqlbuild/compiler/python_nodes/main/_graph.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Public Python-node graph construction entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.python_nodes._helpers.user_inventory import build_python_node_graph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph - - -def build_discovered_python_node_graph( - *, discovered_inputs: DiscoveredProjectInputs -) -> PythonNodeGraph: - """Build the internal executable Python-node graph from discovered inputs.""" - - return build_python_node_graph(discovered_inputs=discovered_inputs) diff --git a/src/sqlbuild/compiler/python_nodes/main/_run_selection.py b/src/sqlbuild/compiler/python_nodes/main/_run_selection.py deleted file mode 100644 index c78145aebb..0000000000 --- a/src/sqlbuild/compiler/python_nodes/main/_run_selection.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Run-command Python-node selection entrypoint.""" - -from __future__ import annotations - -from sqlbuild._native import compiled_graph_keys, resolve_python_sql_run_selection -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes._helpers.user_inventory import build_python_node_graph -from sqlbuild.compiler.python_nodes.models import PythonSqlRunSelection - - -def resolve_python_sql_run_selection_from_inputs( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - project_graph: ProjectGraph, - discovered_inputs: DiscoveredProjectInputs, -) -> PythonSqlRunSelection: - """Resolve run selectors using discovered Python-node project inputs.""" - - sql_keys, python_node_names = resolve_python_sql_run_selection( - project_graph=project_graph, - python_graph=build_python_node_graph(discovered_inputs=discovered_inputs), - select=select, - exclude=exclude, - ) - return PythonSqlRunSelection( - sql_keys=compiled_graph_keys(sql_keys), python_node_names=frozenset(python_node_names) - ) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py index d47675a819..ce7888d745 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py @@ -10,7 +10,11 @@ from types import MappingProxyType, ModuleType from typing import Any, cast -from sqlbuild._native import compiled_graph_keys, resolve_python_sql_run_selection +from sqlbuild._native import ( + build_python_node_graph, + compiled_graph_keys, + resolve_python_sql_run_selection, +) from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -35,7 +39,6 @@ ) from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes._helpers.user_inventory import build_python_node_graph from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunSelection from sqlbuild.providers import Provider from sqlbuild.python_nodes.models import SqlResourceRef diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py index 5bc5f99136..e25150d0a8 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py @@ -9,6 +9,7 @@ import pytest +from sqlbuild._native import build_python_node_graph from sqlbuild.compiler.discovery.models import ( DiscoveredAssetFunction, DiscoveredCheckFunction, @@ -18,10 +19,6 @@ DiscoveredTaskFunction, ) from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.python_nodes._helpers.user_inventory import ( - build_python_node_graph, - build_python_nodes, -) from sqlbuild.compiler.python_nodes.models import ( DiscoveredPythonNode, PythonNodeGraph, @@ -271,63 +268,67 @@ def test_given_fully_configured_python_nodes_when_building_then_identity_config_ retry: RetryPolicy = RetryPolicy(max_attempts=2, retry_on=RuntimeError, jitter=False) columns: tuple[SourceColumnEntry, ...] = (SourceColumnEntry(name="order_id", type="INTEGER"),) - nodes: tuple[DiscoveredPythonNode, ...] = build_python_nodes( - loader_functions=( - DiscoveredLoaderFunction( - file_path=Path("/project/loaders/events.py"), - relative_path=Path("python/loaders/events.py"), - name="load_events", - function=load_events, - destination="raw.events", - cursor_column="updated_at", - unique_key=("event_id",), - columns=columns, - contract="enforced", - connection_mode=LoaderConnectionMode.EXTERNAL, + nodes: tuple[DiscoveredPythonNode, ...] = build_python_node_graph( + discovered_inputs=DiscoveredProjectInputs( + project_config=ProjectConfig(name="demo", adapter="duckdb"), + local_config=LocalConfig(), + loader_functions=( + DiscoveredLoaderFunction( + file_path=Path("/project/loaders/events.py"), + relative_path=Path("python/loaders/events.py"), + name="load_events", + function=load_events, + destination="raw.events", + cursor_column="updated_at", + unique_key=("event_id",), + columns=columns, + contract="enforced", + connection_mode=LoaderConnectionMode.EXTERNAL, + ), ), - ), - task_functions=( - DiscoveredTaskFunction( - file_path=Path("/project/tasks/orders.py"), - relative_path=Path("python/tasks/orders.py"), - name="prepare_orders", - function=prepare_orders, - tags=("orders",), - group="ingestion", - description="Prepare order export inputs.", - meta={"owner": "data-eng"}, - retry=retry, + task_functions=( + DiscoveredTaskFunction( + file_path=Path("/project/tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), + name="prepare_orders", + function=prepare_orders, + tags=("orders",), + group="ingestion", + description="Prepare order export inputs.", + meta={"owner": "data-eng"}, + retry=retry, + ), ), - ), - asset_functions=( - DiscoveredAssetFunction( - file_path=Path("/project/assets/orders.py"), - relative_path=Path("python/assets/orders.py"), - name="export_orders", - function=export_orders, - tags=("orders",), - group="exports", - description="Export orders.", - meta={"owner": "data-eng"}, - columns=columns, - retry=retry, + asset_functions=( + DiscoveredAssetFunction( + file_path=Path("/project/assets/orders.py"), + relative_path=Path("python/assets/orders.py"), + name="export_orders", + function=export_orders, + tags=("orders",), + group="exports", + description="Export orders.", + meta={"owner": "data-eng"}, + columns=columns, + retry=retry, + ), ), - ), - check_functions=( - DiscoveredCheckFunction( - file_path=Path("/project/checks/orders.py"), - relative_path=Path("python/checks/orders.py"), - name="check_orders_export", - function=check_orders_export, - depends_on=(export_orders,), - severity=PythonCheckSeverity.WARN, - tags=("orders",), - group="exports", - description="Check exported orders.", - meta={"owner": "data-eng"}, + check_functions=( + DiscoveredCheckFunction( + file_path=Path("/project/checks/orders.py"), + relative_path=Path("python/checks/orders.py"), + name="check_orders_export", + function=check_orders_export, + depends_on=(export_orders,), + severity=PythonCheckSeverity.WARN, + tags=("orders",), + group="exports", + description="Check exported orders.", + meta={"owner": "data-eng"}, + ), ), - ), - ) + ) + ).nodes assert { f"{node.kind.value}:{node.name}": tuple( diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index df86a5e314..1d79f6747a 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any, ClassVar, cast -from sqlbuild._native import CostContext, build_python_sql_run_lifecycle +from sqlbuild._native import CostContext, build_python_node_graph, build_python_sql_run_lifecycle from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.assets import AssetContext, asset @@ -23,7 +23,6 @@ DiscoveredProjectInputs, DiscoveredTaskFunction, ) -from sqlbuild.compiler.python_nodes._helpers.user_inventory import build_python_node_graph from sqlbuild.compiler.python_nodes.models import ( DiscoveredPythonNode, PythonNodeGraph, From 19c68dd63436d170ab278459e04d67717d953be0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:13:28 +0100 Subject: [PATCH 07/11] refactor: plan macro module loading natively The synthetic package digest, parent package set and dependency-first execution order of project macro modules come from a native planner; Python keeps only the shim that installs modules and executes user macro code. --- .../src/macro_modules/_helpers/load_plan.rs | 71 +++++++++++++++++++ .../src/macro_modules/_helpers/mod.rs | 1 + .../src/macro_modules/constants.rs | 4 ++ .../src/macro_modules/main/macro_load_plan.rs | 15 ++++ .../src/macro_modules/main/mod.rs | 1 + .../src/macro_modules/models.rs | 17 +++++ .../bindings/_helpers/render/macro_modules.rs | 27 +++++++ src/sqlbuild/_native.pyi | 3 + .../compile/_helpers/render/user_macros.py | 51 ++++--------- 9 files changed, 153 insertions(+), 37 deletions(-) create mode 100644 crates/sqlbuild-pysource/src/macro_modules/_helpers/load_plan.rs create mode 100644 crates/sqlbuild-pysource/src/macro_modules/main/macro_load_plan.rs diff --git a/crates/sqlbuild-pysource/src/macro_modules/_helpers/load_plan.rs b/crates/sqlbuild-pysource/src/macro_modules/_helpers/load_plan.rs new file mode 100644 index 0000000000..b9f3be44fb --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/_helpers/load_plan.rs @@ -0,0 +1,71 @@ +//! The synthetic package name, parent packages and dependency-first order of macro modules. + +use std::cmp::Reverse; +use std::collections::{BTreeSet, BinaryHeap, HashMap}; + +use sha2::{Digest, Sha256}; + +use crate::macro_modules::constants::{SYNTHETIC_DIGEST_LENGTH, SYNTHETIC_PACKAGE_PREFIX}; +use crate::macro_modules::models::MacroModuleSpec; + +/// The root package named by a digest of every module's name, path and contents. +pub(crate) fn synthetic_root(modules: &[MacroModuleSpec]) -> String { + let mut by_name: Vec<&MacroModuleSpec> = modules.iter().collect(); + by_name.sort_by(|left, right| left.name.cmp(&right.name)); + let joined: String = by_name + .iter() + .map(|module| format!("{}\0{}\0{}", module.name, module.file_path, module.contents)) + .collect::>() + .join("\0"); + let digest: String = format!("{:x}", Sha256::digest(joined.as_bytes())); + format!( + "{SYNTHETIC_PACKAGE_PREFIX}{}", + &digest[..SYNTHETIC_DIGEST_LENGTH] + ) +} + +/// The root and every parent package of a module, shallowest first. +pub(crate) fn parent_packages(root: &str, modules: &[MacroModuleSpec]) -> Vec { + let mut packages: BTreeSet<(usize, String)> = BTreeSet::from([(0, root.to_owned())]); + for module in modules { + let parts: Vec<&str> = module.name.split('.').collect(); + for index in 1..parts.len() { + let package: String = format!("{root}.{}", parts[..index].join(".")); + packages.insert((package.matches('.').count(), package)); + } + } + packages.into_iter().map(|(_, package)| package).collect() +} + +/// Kahn order over module dependencies, taking the smallest ready name first. +pub(crate) fn load_order(modules: &[MacroModuleSpec]) -> Vec { + let mut remaining: HashMap<&str, usize> = HashMap::new(); + let mut dependents: HashMap<&str, Vec<&str>> = HashMap::new(); + for module in modules { + remaining.insert(&module.name, module.dependencies.len()); + for dependency in &module.dependencies { + dependents + .entry(dependency.as_str()) + .or_default() + .push(&module.name); + } + } + let mut ready: BinaryHeap> = remaining + .iter() + .filter(|(_, count)| **count == 0) + .map(|(name, _)| Reverse(*name)) + .collect(); + let mut order: Vec = Vec::new(); + while let Some(Reverse(name)) = ready.pop() { + order.push(name.to_owned()); + for dependent in dependents.get(name).into_iter().flatten() { + if let Some(count) = remaining.get_mut(dependent) { + *count -= 1; + if *count == 0 { + ready.push(Reverse(dependent)); + } + } + } + } + order +} diff --git a/crates/sqlbuild-pysource/src/macro_modules/_helpers/mod.rs b/crates/sqlbuild-pysource/src/macro_modules/_helpers/mod.rs index 18d81a3751..157f515dc0 100644 --- a/crates/sqlbuild-pysource/src/macro_modules/_helpers/mod.rs +++ b/crates/sqlbuild-pysource/src/macro_modules/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod declarations; pub(crate) mod function_calls; +pub(crate) mod load_plan; pub(crate) mod module_analysis; pub(crate) mod module_names; pub(crate) mod project_analysis; diff --git a/crates/sqlbuild-pysource/src/macro_modules/constants.rs b/crates/sqlbuild-pysource/src/macro_modules/constants.rs index 2fdb0bd3ab..d3ceef59cc 100644 --- a/crates/sqlbuild-pysource/src/macro_modules/constants.rs +++ b/crates/sqlbuild-pysource/src/macro_modules/constants.rs @@ -12,3 +12,7 @@ pub(crate) const PRIVATE_PREFIX: &str = "_"; pub(crate) const PATH_ROOTS: [char; 2] = ['/', '\\']; /// Separator of the names in a reported call or import cycle. pub(crate) const CYCLE_SEPARATOR: &str = " -> "; +/// Prefix of the synthetic package macro modules are executed under. +pub(crate) const SYNTHETIC_PACKAGE_PREFIX: &str = "_sqlbuild_project_macros_"; +/// Hex digits of the content digest naming the synthetic package. +pub(crate) const SYNTHETIC_DIGEST_LENGTH: usize = 16; diff --git a/crates/sqlbuild-pysource/src/macro_modules/main/macro_load_plan.rs b/crates/sqlbuild-pysource/src/macro_modules/main/macro_load_plan.rs new file mode 100644 index 0000000000..343befc1cc --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/main/macro_load_plan.rs @@ -0,0 +1,15 @@ +//! The synthetic package and dependency-first execution order of project macro modules. + +use crate::macro_modules::_helpers::load_plan::{load_order, parent_packages, synthetic_root}; +use crate::macro_modules::models::{MacroLoadPlan, MacroModuleSpec}; + +/// Root package named by a content digest, parent packages, and Kahn order with smallest name first. +#[must_use] +pub fn macro_load_plan(modules: &[MacroModuleSpec]) -> MacroLoadPlan { + let root: String = synthetic_root(modules); + MacroLoadPlan { + packages: parent_packages(&root, modules), + order: load_order(modules), + root, + } +} diff --git a/crates/sqlbuild-pysource/src/macro_modules/main/mod.rs b/crates/sqlbuild-pysource/src/macro_modules/main/mod.rs index bf4f23865d..fdfda80311 100644 --- a/crates/sqlbuild-pysource/src/macro_modules/main/mod.rs +++ b/crates/sqlbuild-pysource/src/macro_modules/main/mod.rs @@ -1 +1,2 @@ pub mod inventory_macro_modules; +pub mod macro_load_plan; diff --git a/crates/sqlbuild-pysource/src/macro_modules/models.rs b/crates/sqlbuild-pysource/src/macro_modules/models.rs index 64f5eea25f..151af0b39f 100644 --- a/crates/sqlbuild-pysource/src/macro_modules/models.rs +++ b/crates/sqlbuild-pysource/src/macro_modules/models.rs @@ -64,3 +64,20 @@ pub struct MacroModuleInventory { /// Every module in discovery order, present only when the strict analysis passed. pub modules: Option>, } + +/// One macro module to load: its module name, file path text, contents and module dependencies. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MacroModuleSpec { + pub name: String, + pub file_path: String, + pub contents: String, + pub dependencies: Vec, +} + +/// The synthetic package root, its packages shallowest first, and the module execution order. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MacroLoadPlan { + pub root: String, + pub packages: Vec, + pub order: Vec, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs index 8b561d1a36..d19447b762 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs @@ -3,10 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_pysource::macro_modules::main::inventory_macro_modules::inventory_macro_modules as inventory; +use sqlbuild_pysource::macro_modules::main::macro_load_plan::macro_load_plan; use sqlbuild_pysource::macro_modules::models::{ MacroExport, MacroFault, MacroModuleInventory, MacroModuleLoad, MacroModulePath, MacroModuleSource, }; +use sqlbuild_pysource::macro_modules::models::{MacroLoadPlan, MacroModuleSpec}; use sqlbuild_pysource::macro_modules::types::MacroScopeKind; use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_error}; @@ -108,7 +110,32 @@ pub(crate) fn inventory_macro_modules( )) } +/// `(synthetic root, packages shallowest first, module execution order)` of `(name, path, contents, dependencies)` rows. +#[pyfunction] +fn macro_module_load_plan( + modules: Vec<(String, String, String, Vec)>, +) -> (String, Vec, Vec) { + let specs: Vec = modules + .into_iter() + .map( + |(name, file_path, contents, dependencies)| MacroModuleSpec { + name, + file_path, + contents, + dependencies, + }, + ) + .collect(); + let MacroLoadPlan { + root, + packages, + order, + } = macro_load_plan(&specs); + (root, packages, order) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(inventory_macro_modules, module)?)?; + module.add_function(wrap_pyfunction!(macro_module_load_plan, module)?)?; Ok(()) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index c5d57dae9a..9b5ed2d7bf 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4946,6 +4946,9 @@ def load_sqlbuild_dag( dag: Mapping[str, Any] | str | Path, *, error_class: type[Exception] ) -> Mapping[str, Any]: ... def build_python_node_graph(*, discovered_inputs: DiscoveredProjectInputs) -> PythonNodeGraph: ... +def macro_module_load_plan( + modules: Sequence[tuple[str, str, str, Sequence[str]]], +) -> tuple[str, list[str], list[str]]: ... def fit_artifact_logical_name( *, logical_name: str, fixed_prefix: str, identifier_limit: int, artifact_label: str ) -> str: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py index 7473eecc9d..a6c0b0fcaa 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py @@ -3,8 +3,6 @@ from __future__ import annotations import ast -import hashlib -import heapq import inspect import sys import threading @@ -42,7 +40,6 @@ from sqlbuild.sql_values.models import SqlValue from sqlbuild.sql_values.types import SqlValueKind -_SYNTHETIC_PACKAGE_PREFIX: str = "_sqlbuild_project_macros_" _OBJECT_ENTRY_LENGTH: int = 2 _MACRO_MODULE_LOAD_LOCK: threading.RLock = threading.RLock() @@ -268,25 +265,22 @@ def _load_macro_modules(*, analyses: dict[str, _MacroModuleLoad]) -> dict[str, M def _load_macro_modules_locked(*, analyses: dict[str, _MacroModuleLoad]) -> dict[str, ModuleType]: - digest: str = hashlib.sha256( - "\0".join( - f"{name}\0{analyses[name].file.file_path}\0{analyses[name].file.contents}" - for name in sorted(analyses) - ).encode() - ).hexdigest()[:16] - root: str = f"{_SYNTHETIC_PACKAGE_PREFIX}{digest}" + root, package_names, load_order = _native.macro_module_load_plan( + [ + ( + name, + str(analysis.file.file_path), + analysis.file.contents, + list(analysis.module_dependencies), + ) + for name, analysis in analyses.items() + ] + ) synthetic_names: list[str] = [] previous_modules: dict[str, ModuleType] = {} modules: dict[str, ModuleType] = {} try: - package_names: set[str] = {root} - for name in analyses: - parts: list[str] = name.split(".")[:-1] - package_names.update( - f"{root}.{'/'.join(parts[:index]).replace('/', '.')}" - for index in range(1, len(parts) + 1) - ) - for package_name in sorted(package_names, key=lambda item: item.count(".")): + for package_name in package_names: package: ModuleType = ModuleType(package_name) package.__package__ = package_name package.__path__ = [] # type: ignore[attr-defined] @@ -296,21 +290,8 @@ def _load_macro_modules_locked(*, analyses: dict[str, _MacroModuleLoad]) -> dict sys.modules[package_name] = package synthetic_names.append(package_name) - remaining_dependencies: dict[str, int] = { - name: len(analysis.module_dependencies) for name, analysis in analyses.items() - } - dependents: dict[str, list[str]] = {} - for name, analysis in analyses.items(): - for dependency in analysis.module_dependencies: - dependents.setdefault(dependency, []).append(name) - ready: list[str] = [ - name - for name, dependency_count in remaining_dependencies.items() - if dependency_count == 0 - ] - heapq.heapify(ready) - while ready: - name: str = heapq.heappop(ready) + name: str + for name in load_order: analysis: _MacroModuleLoad = analyses[name] synthetic_name: str = f"{root}.{name}" module: ModuleType = ModuleType(synthetic_name) @@ -331,10 +312,6 @@ def _load_macro_modules_locked(*, analyses: dict[str, _MacroModuleLoad]) -> dict f"Failed to load macros from '{analysis.file.file_path}': {error}" ) from error modules[name] = module - for dependent in dependents.get(name, ()): - remaining_dependencies[dependent] -= 1 - if remaining_dependencies[dependent] == 0: - heapq.heappush(ready, dependent) return modules finally: for synthetic_name in reversed(synthetic_names): From ddf39dff5bcb06971a3e35bc237fed3748735f59 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:18:48 +0100 Subject: [PATCH 08/11] refactor: satisfy clippy in the native Python-node graph and run selection callers --- .../_helpers/compile_driver/plan_result.rs | 7 ++---- .../_helpers/graph/python_node_graph.rs | 24 +++++++------------ 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 19b65a0be8..f023e6ef3d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -58,11 +58,8 @@ pub(crate) fn pipeline_result<'py>( &native.getattr("build_project_graph_from_compiled_project")?, &[("project", project.clone())], )?; - run_selection = run_selection_from_inputs( - (select.extract()?, exclude.extract()?), - &graph, - &discovered, - )?; + run_selection = + run_selection_from_inputs((select.extract()?, exclude.extract()?), &graph, discovered)?; selected_sql_keys = run_selection.getattr("sql_keys")?; selected_python_node_names = run_selection.getattr("python_node_names")?; python_read_source_names = call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs index fa8f0d1cee..c3ff893545 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs @@ -238,14 +238,7 @@ pub(crate) fn python_node_graph<'py>( .zip(upstream_names(py, &nodes, &vocabulary.sql_ref)?) .map(|(node, upstream)| inventory_node(node, upstream)) .collect::>()?; - let InventoryIndexes { - edges, - upstream, - downstream, - tags, - paths, - } = graph_indexes(&inventory); - graph_record(py, &nodes, (edges, upstream, downstream, tags, paths)) + graph_record(py, &nodes, graph_indexes(&inventory)) } type NameRows = Vec<(String, Vec)>; @@ -253,14 +246,15 @@ type NameRows = Vec<(String, Vec)>; fn graph_record<'py>( py: Python<'py>, nodes: &[Bound<'py, PyAny>], - (edges, upstream, downstream, tags, paths): ( - Vec<(String, String)>, - NameRows, - NameRows, - NameRows, - Vec<(String, String)>, - ), + indexes: InventoryIndexes, ) -> PyResult> { + let InventoryIndexes { + edges, + upstream, + downstream, + tags, + paths, + } = indexes; let edge_class: Bound<'py, PyAny> = imported(py, NODE_MODELS_MODULE, "PythonNodeDependencyEdge")?; let dependency_edges: Vec> = edges From 816e1cf20c44bc61b260ab76f0f320c714b91708 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:32:39 +0100 Subject: [PATCH 09/11] style: format integrated compile-path bindings --- .../bindings/_helpers/planner_registration/registration.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs index 1fcf7cf738..62cfa9cd58 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs @@ -3,7 +3,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::{ - planner_changes, planner_entries, planner_identity, planner_migrations, planner_models, planner_phases, planner_resolution, planner_scenario, planner_scope, planner_selection, planner_strategy, planner_warehouse, relation_naming, + planner_changes, planner_entries, planner_identity, planner_migrations, planner_models, + planner_phases, planner_resolution, planner_scenario, planner_scope, planner_selection, + planner_strategy, planner_warehouse, relation_naming, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { From 1337dd8da4537c2cc76e27c66b24b1e022cad25f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:41:03 +0100 Subject: [PATCH 10/11] chore: drop stale imports and group registration after compile-path integration --- crates/sqlbuild-python/src/bindings/main/register.rs | 2 +- src/sqlbuild/_native.pyi | 8 +------- .../planner/_helpers/sql_test_assembly/test_plan_test.py | 2 -- .../sqlbuild/executor/pipeline/_helpers/test_auditing.py | 5 ++--- 4 files changed, 4 insertions(+), 13 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 4ff394572a..da39e9d00a 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -28,7 +28,6 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { analysis::normalization::register(module)?; analysis::queries::register(module)?; analysis::sql_tests::register(module)?; - analysis::sql_test_comparisons::register(module)?; sqltext::registration::register(module)?; boundary::oracles::register(module)?; boundary::expected_errors::register(module)?; @@ -56,6 +55,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { state_rows::registration::register(module)?; diagnostics::redaction::register(module)?; diagnostics::trackers::register(module)?; + analysis::sql_test_comparisons::register(module)?; Ok(()) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 9b5ed2d7bf..c41205c948 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -176,9 +176,6 @@ from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( AuditPlanEntry, ChainStep, - BackfillResult, - ChainStep, - ChangeDetectionResult, CloneSourcePlanEntry, ColumnMigrationPlanEntry, CursorBounds, @@ -332,10 +329,7 @@ from sqlbuild.lint.models import ( PreparedLintFiles, RuleFixResult, ) -from sqlbuild.lint.types import ( - RelationKeyRow, -) -from sqlbuild.lint.types import RelationKeyRow, RelationKeys +from sqlbuild.lint.types import RelationKeyRow from sqlbuild.microbatches.models import MicrobatchInterval from sqlbuild.presentation.models import DisplayOptions from sqlbuild.presentation.types import TransientLineOwner diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index 94d3b06183..6f47e2106d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -42,8 +42,6 @@ PlanOutput, SqlTestPlanEntry, ) -from sqlbuild.compiler.planner.types import WarningSeverity -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( AssertionChainCteErrorTestCase, NativePlanningDifferentialTestCase, diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py index 30097490c8..c7aefb0ff7 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py @@ -21,8 +21,8 @@ from sqlbuild._native import ( AuditProgressReporter, CostContext, - audit_resource_id, StatementLedgerEntry, + audit_resource_id, read_statement_ledger, record_statement, ) @@ -37,8 +37,7 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput -from sqlbuild.cost.models import CostResourceContext, StatementLedgerEntry -from sqlbuild.executor.auditing.main.resource_id import audit_resource_id +from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.pipeline._helpers import auditing from sqlbuild.executor.pipeline.exceptions import AuditExecutionError, AuditOutcomeError From 8270e51b2616f098f5b392a7926c07ba154ea8e3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:14:55 +0100 Subject: [PATCH 11/11] refactor: drop unused executor testing constants --- src/sqlbuild/executor/testing/constants.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/sqlbuild/executor/testing/constants.py b/src/sqlbuild/executor/testing/constants.py index 1ef960c924..5cabd16b52 100644 --- a/src/sqlbuild/executor/testing/constants.py +++ b/src/sqlbuild/executor/testing/constants.py @@ -6,6 +6,3 @@ SQL_TEST_ENTRY_MISSING_CODE: str = "T004" SQL_TEST_EXPECTED_COLUMNS_CODE: str = "T005" SQL_TEST_SIZE_BYTES: str = "bytes" - -SQL_TEST_NATIVE_RENDER_WORKERS: int = 4 -SQL_TEST_IDENTIFIER_QUOTES: dict[str, str] = {'"': '"', "`": "`", "[": "]"}